2020-06-01 17:16:09 +00:00
|
|
|
#include <Storages/StorageS3Settings.h>
|
|
|
|
|
2022-09-15 07:45:28 +00:00
|
|
|
#include <IO/S3Common.h>
|
|
|
|
|
2020-06-01 17:16:09 +00:00
|
|
|
#include <Poco/Util/AbstractConfiguration.h>
|
2022-12-20 21:33:54 +00:00
|
|
|
#include <Common/NamedCollections/NamedCollections.h>
|
2020-06-01 17:16:09 +00:00
|
|
|
#include <Common/Exception.h>
|
2022-11-17 16:35:04 +00:00
|
|
|
#include <Common/Throttler.h>
|
2023-12-23 13:46:21 +00:00
|
|
|
#include <Common/formatReadable.h>
|
2022-04-03 22:33:59 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2021-01-07 03:42:39 +00:00
|
|
|
#include <boost/algorithm/string/predicate.hpp>
|
|
|
|
|
2020-06-01 17:16:09 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
namespace ErrorCodes
|
2020-06-01 17:16:09 +00:00
|
|
|
{
|
2022-12-17 14:09:53 +00:00
|
|
|
extern const int INVALID_SETTING_VALUE;
|
|
|
|
}
|
2020-06-01 17:16:09 +00:00
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(const Settings & settings)
|
|
|
|
{
|
2022-12-17 16:02:34 +00:00
|
|
|
updateFromSettingsImpl(settings, false);
|
2022-12-17 14:09:53 +00:00
|
|
|
validate();
|
|
|
|
}
|
|
|
|
|
|
|
|
S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(
|
|
|
|
const Poco::Util::AbstractConfiguration & config,
|
2022-12-18 12:31:17 +00:00
|
|
|
const String & config_prefix,
|
|
|
|
const Settings & settings,
|
|
|
|
String setting_name_prefix)
|
2022-12-17 14:09:53 +00:00
|
|
|
: PartUploadSettings(settings)
|
|
|
|
{
|
2022-12-18 12:31:17 +00:00
|
|
|
String key = config_prefix + "." + setting_name_prefix;
|
2023-04-05 20:21:19 +00:00
|
|
|
strict_upload_part_size = config.getUInt64(key + "strict_upload_part_size", strict_upload_part_size);
|
2022-12-18 12:31:17 +00:00
|
|
|
min_upload_part_size = config.getUInt64(key + "min_upload_part_size", min_upload_part_size);
|
|
|
|
max_upload_part_size = config.getUInt64(key + "max_upload_part_size", max_upload_part_size);
|
|
|
|
upload_part_size_multiply_factor = config.getUInt64(key + "upload_part_size_multiply_factor", upload_part_size_multiply_factor);
|
|
|
|
upload_part_size_multiply_parts_count_threshold = config.getUInt64(key + "upload_part_size_multiply_parts_count_threshold", upload_part_size_multiply_parts_count_threshold);
|
2023-05-21 10:42:28 +00:00
|
|
|
max_inflight_parts_for_one_file = config.getUInt64(key + "max_inflight_parts_for_one_file", max_inflight_parts_for_one_file);
|
2022-12-18 12:31:17 +00:00
|
|
|
max_part_number = config.getUInt64(key + "max_part_number", max_part_number);
|
|
|
|
max_single_part_upload_size = config.getUInt64(key + "max_single_part_upload_size", max_single_part_upload_size);
|
|
|
|
max_single_operation_copy_size = config.getUInt64(key + "max_single_operation_copy_size", max_single_operation_copy_size);
|
2022-12-17 14:09:53 +00:00
|
|
|
|
2022-12-30 13:05:01 +00:00
|
|
|
/// This configuration is only applicable to s3. Other types of object storage are not applicable or have different meanings.
|
|
|
|
storage_class_name = config.getString(config_prefix + ".s3_storage_class", storage_class_name);
|
|
|
|
storage_class_name = Poco::toUpperInPlace(storage_class_name);
|
2022-12-17 14:09:53 +00:00
|
|
|
|
|
|
|
validate();
|
|
|
|
}
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
S3Settings::RequestSettings::PartUploadSettings::PartUploadSettings(const NamedCollection & collection)
|
|
|
|
{
|
2023-04-05 20:21:19 +00:00
|
|
|
strict_upload_part_size = collection.getOrDefault<UInt64>("strict_upload_part_size", strict_upload_part_size);
|
2022-12-17 17:12:04 +00:00
|
|
|
min_upload_part_size = collection.getOrDefault<UInt64>("min_upload_part_size", min_upload_part_size);
|
2023-04-05 20:21:19 +00:00
|
|
|
max_single_part_upload_size = collection.getOrDefault<UInt64>("max_single_part_upload_size", max_single_part_upload_size);
|
2022-12-17 17:12:04 +00:00
|
|
|
upload_part_size_multiply_factor = collection.getOrDefault<UInt64>("upload_part_size_multiply_factor", upload_part_size_multiply_factor);
|
|
|
|
upload_part_size_multiply_parts_count_threshold = collection.getOrDefault<UInt64>("upload_part_size_multiply_parts_count_threshold", upload_part_size_multiply_parts_count_threshold);
|
2023-05-21 10:42:28 +00:00
|
|
|
max_inflight_parts_for_one_file = collection.getOrDefault<UInt64>("max_inflight_parts_for_one_file", max_inflight_parts_for_one_file);
|
2022-12-17 17:12:04 +00:00
|
|
|
|
2022-12-30 13:05:01 +00:00
|
|
|
/// This configuration is only applicable to s3. Other types of object storage are not applicable or have different meanings.
|
|
|
|
storage_class_name = collection.getOrDefault<String>("s3_storage_class", storage_class_name);
|
|
|
|
storage_class_name = Poco::toUpperInPlace(storage_class_name);
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
validate();
|
|
|
|
}
|
|
|
|
|
2022-12-17 16:02:34 +00:00
|
|
|
void S3Settings::RequestSettings::PartUploadSettings::updateFromSettingsImpl(const Settings & settings, bool if_changed)
|
|
|
|
{
|
2023-04-05 20:21:19 +00:00
|
|
|
if (!if_changed || settings.s3_strict_upload_part_size.changed)
|
|
|
|
strict_upload_part_size = settings.s3_strict_upload_part_size;
|
|
|
|
|
2022-12-17 16:02:34 +00:00
|
|
|
if (!if_changed || settings.s3_min_upload_part_size.changed)
|
|
|
|
min_upload_part_size = settings.s3_min_upload_part_size;
|
|
|
|
|
2022-12-17 21:23:08 +00:00
|
|
|
if (!if_changed || settings.s3_max_upload_part_size.changed)
|
|
|
|
max_upload_part_size = settings.s3_max_upload_part_size;
|
|
|
|
|
2022-12-17 16:02:34 +00:00
|
|
|
if (!if_changed || settings.s3_upload_part_size_multiply_factor.changed)
|
|
|
|
upload_part_size_multiply_factor = settings.s3_upload_part_size_multiply_factor;
|
|
|
|
|
|
|
|
if (!if_changed || settings.s3_upload_part_size_multiply_parts_count_threshold.changed)
|
|
|
|
upload_part_size_multiply_parts_count_threshold = settings.s3_upload_part_size_multiply_parts_count_threshold;
|
|
|
|
|
2023-05-17 13:40:36 +00:00
|
|
|
if (!if_changed || settings.s3_max_inflight_parts_for_one_file.changed)
|
2023-05-21 10:42:28 +00:00
|
|
|
max_inflight_parts_for_one_file = settings.s3_max_inflight_parts_for_one_file;
|
2023-05-17 13:40:36 +00:00
|
|
|
|
2022-12-17 16:02:34 +00:00
|
|
|
if (!if_changed || settings.s3_max_single_part_upload_size.changed)
|
|
|
|
max_single_part_upload_size = settings.s3_max_single_part_upload_size;
|
|
|
|
}
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
void S3Settings::RequestSettings::PartUploadSettings::validate()
|
|
|
|
{
|
|
|
|
static constexpr size_t min_upload_part_size_limit = 5 * 1024 * 1024;
|
2023-04-05 20:21:19 +00:00
|
|
|
if (strict_upload_part_size && strict_upload_part_size < min_upload_part_size_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting strict_upload_part_size has invalid value {} which is less than the s3 API limit {}",
|
|
|
|
ReadableSize(strict_upload_part_size), ReadableSize(min_upload_part_size_limit));
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
if (min_upload_part_size < min_upload_part_size_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting min_upload_part_size has invalid value {} which is less than the s3 API limit {}",
|
|
|
|
ReadableSize(min_upload_part_size), ReadableSize(min_upload_part_size_limit));
|
|
|
|
|
|
|
|
static constexpr size_t max_upload_part_size_limit = 5ull * 1024 * 1024 * 1024;
|
|
|
|
if (max_upload_part_size > max_upload_part_size_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting max_upload_part_size has invalid value {} which is grater than the s3 API limit {}",
|
|
|
|
ReadableSize(max_upload_part_size), ReadableSize(max_upload_part_size_limit));
|
|
|
|
|
|
|
|
if (max_single_part_upload_size > max_upload_part_size_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting max_single_part_upload_size has invalid value {} which is grater than the s3 API limit {}",
|
|
|
|
ReadableSize(max_single_part_upload_size), ReadableSize(max_upload_part_size_limit));
|
|
|
|
|
|
|
|
if (max_single_operation_copy_size > max_upload_part_size_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting max_single_operation_copy_size has invalid value {} which is grater than the s3 API limit {}",
|
|
|
|
ReadableSize(max_single_operation_copy_size), ReadableSize(max_upload_part_size_limit));
|
|
|
|
|
|
|
|
if (max_upload_part_size < min_upload_part_size)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting max_upload_part_size ({}) can't be less than setting min_upload_part_size {}",
|
|
|
|
ReadableSize(max_upload_part_size), ReadableSize(min_upload_part_size));
|
|
|
|
|
|
|
|
if (!upload_part_size_multiply_factor)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
2023-01-23 22:16:16 +00:00
|
|
|
"Setting upload_part_size_multiply_factor cannot be zero");
|
2022-12-17 14:09:53 +00:00
|
|
|
|
|
|
|
if (!upload_part_size_multiply_parts_count_threshold)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
2023-01-23 22:16:16 +00:00
|
|
|
"Setting upload_part_size_multiply_parts_count_threshold cannot be zero");
|
2022-12-17 14:09:53 +00:00
|
|
|
|
|
|
|
if (!max_part_number)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
2023-01-23 22:16:16 +00:00
|
|
|
"Setting max_part_number cannot be zero");
|
2022-12-17 14:09:53 +00:00
|
|
|
|
|
|
|
static constexpr size_t max_part_number_limit = 10000;
|
|
|
|
if (max_part_number > max_part_number_limit)
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting max_part_number has invalid value {} which is grater than the s3 API limit {}",
|
|
|
|
ReadableSize(max_part_number), ReadableSize(max_part_number_limit));
|
|
|
|
|
|
|
|
size_t maybe_overflow;
|
|
|
|
if (common::mulOverflow(max_upload_part_size, upload_part_size_multiply_factor, maybe_overflow))
|
|
|
|
throw Exception(
|
2023-01-23 21:13:58 +00:00
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
|
|
|
"Setting upload_part_size_multiply_factor is too big ({}). "
|
|
|
|
"Multiplication to max_upload_part_size ({}) will cause integer overflow",
|
|
|
|
ReadableSize(max_part_number), ReadableSize(max_part_number_limit));
|
2022-12-17 14:09:53 +00:00
|
|
|
|
2023-01-04 16:23:22 +00:00
|
|
|
std::unordered_set<String> storage_class_names {"STANDARD", "INTELLIGENT_TIERING"};
|
|
|
|
if (!storage_class_name.empty() && !storage_class_names.contains(storage_class_name))
|
2022-12-30 13:05:01 +00:00
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::INVALID_SETTING_VALUE,
|
2023-01-04 16:23:22 +00:00
|
|
|
"Setting storage_class has invalid value {} which only supports STANDARD and INTELLIGENT_TIERING",
|
2022-12-30 13:05:01 +00:00
|
|
|
storage_class_name);
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
/// TODO: it's possible to set too small limits. We can check that max possible object size is not too small.
|
|
|
|
}
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
S3Settings::RequestSettings::RequestSettings(const Settings & settings)
|
|
|
|
: upload_settings(settings)
|
|
|
|
{
|
2022-12-17 16:02:34 +00:00
|
|
|
updateFromSettingsImpl(settings, false);
|
|
|
|
}
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
S3Settings::RequestSettings::RequestSettings(const NamedCollection & collection)
|
|
|
|
: upload_settings(collection)
|
2022-12-17 16:02:34 +00:00
|
|
|
{
|
2022-12-17 17:12:04 +00:00
|
|
|
max_single_read_retries = collection.getOrDefault<UInt64>("max_single_read_retries", max_single_read_retries);
|
|
|
|
max_connections = collection.getOrDefault<UInt64>("max_connections", max_connections);
|
2023-01-20 19:10:23 +00:00
|
|
|
list_object_keys_size = collection.getOrDefault<UInt64>("list_object_keys_size", list_object_keys_size);
|
2023-06-27 14:20:27 +00:00
|
|
|
allow_native_copy = collection.getOrDefault<bool>("allow_native_copy", allow_native_copy);
|
2023-02-04 14:11:14 +00:00
|
|
|
throw_on_zero_files_match = collection.getOrDefault<bool>("throw_on_zero_files_match", throw_on_zero_files_match);
|
2022-12-17 16:02:34 +00:00
|
|
|
}
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
S3Settings::RequestSettings::RequestSettings(
|
|
|
|
const Poco::Util::AbstractConfiguration & config,
|
2022-12-18 12:31:17 +00:00
|
|
|
const String & config_prefix,
|
|
|
|
const Settings & settings,
|
|
|
|
String setting_name_prefix)
|
|
|
|
: upload_settings(config, config_prefix, settings, setting_name_prefix)
|
2022-12-17 14:09:53 +00:00
|
|
|
{
|
2022-12-18 12:31:17 +00:00
|
|
|
String key = config_prefix + "." + setting_name_prefix;
|
|
|
|
max_single_read_retries = config.getUInt64(key + "max_single_read_retries", settings.s3_max_single_read_retries);
|
|
|
|
max_connections = config.getUInt64(key + "max_connections", settings.s3_max_connections);
|
|
|
|
check_objects_after_upload = config.getBool(key + "check_objects_after_upload", settings.s3_check_objects_after_upload);
|
2023-01-20 19:10:23 +00:00
|
|
|
list_object_keys_size = config.getUInt64(key + "list_object_keys_size", settings.s3_list_object_keys_size);
|
2023-06-27 14:20:27 +00:00
|
|
|
allow_native_copy = config.getBool(key + "allow_native_copy", allow_native_copy);
|
2023-02-04 14:11:14 +00:00
|
|
|
throw_on_zero_files_match = config.getBool(key + "throw_on_zero_files_match", settings.s3_throw_on_zero_files_match);
|
2023-05-21 10:42:28 +00:00
|
|
|
retry_attempts = config.getUInt64(key + "retry_attempts", settings.s3_retry_attempts);
|
2023-06-19 23:35:35 +00:00
|
|
|
request_timeout_ms = config.getUInt64(key + "request_timeout_ms", settings.s3_request_timeout_ms);
|
2022-04-03 22:33:59 +00:00
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
/// NOTE: it would be better to reuse old throttlers to avoid losing token bucket state on every config reload,
|
|
|
|
/// which could lead to exceeding limit for short time. But it is good enough unless very high `burst` values are used.
|
2022-12-18 12:31:17 +00:00
|
|
|
if (UInt64 max_get_rps = config.getUInt64(key + "max_get_rps", settings.s3_max_get_rps))
|
2022-04-03 22:33:59 +00:00
|
|
|
{
|
2022-12-17 14:09:53 +00:00
|
|
|
size_t default_max_get_burst = settings.s3_max_get_burst
|
|
|
|
? settings.s3_max_get_burst
|
|
|
|
: (Throttler::default_burst_seconds * max_get_rps);
|
2022-04-03 22:33:59 +00:00
|
|
|
|
2022-12-18 12:31:17 +00:00
|
|
|
size_t max_get_burst = config.getUInt64(key + "max_get_burst", default_max_get_burst);
|
2022-08-23 13:12:09 +00:00
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
get_request_throttler = std::make_shared<Throttler>(max_get_rps, max_get_burst);
|
|
|
|
}
|
2022-12-18 12:31:17 +00:00
|
|
|
if (UInt64 max_put_rps = config.getUInt64(key + "max_put_rps", settings.s3_max_put_rps))
|
2022-08-23 13:12:09 +00:00
|
|
|
{
|
2022-12-17 14:09:53 +00:00
|
|
|
size_t default_max_put_burst = settings.s3_max_put_burst
|
|
|
|
? settings.s3_max_put_burst
|
|
|
|
: (Throttler::default_burst_seconds * max_put_rps);
|
|
|
|
|
2022-12-18 12:31:17 +00:00
|
|
|
size_t max_put_burst = config.getUInt64(key + "max_put_burst", default_max_put_burst);
|
2022-12-17 14:09:53 +00:00
|
|
|
|
|
|
|
put_request_throttler = std::make_shared<Throttler>(max_put_rps, max_put_burst);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
void S3Settings::RequestSettings::updateFromSettingsImpl(const Settings & settings, bool if_changed)
|
|
|
|
{
|
|
|
|
if (!if_changed || settings.s3_max_single_read_retries.changed)
|
|
|
|
max_single_read_retries = settings.s3_max_single_read_retries;
|
|
|
|
|
|
|
|
if (!if_changed || settings.s3_max_connections.changed)
|
|
|
|
max_connections = settings.s3_max_connections;
|
|
|
|
|
|
|
|
if (!if_changed || settings.s3_check_objects_after_upload.changed)
|
|
|
|
check_objects_after_upload = settings.s3_check_objects_after_upload;
|
|
|
|
|
|
|
|
if (!if_changed || settings.s3_max_unexpected_write_error_retries.changed)
|
|
|
|
max_unexpected_write_error_retries = settings.s3_max_unexpected_write_error_retries;
|
|
|
|
|
2023-01-20 19:10:23 +00:00
|
|
|
if (!if_changed || settings.s3_list_object_keys_size.changed)
|
|
|
|
list_object_keys_size = settings.s3_list_object_keys_size;
|
|
|
|
|
2022-12-17 17:12:04 +00:00
|
|
|
if ((!if_changed || settings.s3_max_get_rps.changed || settings.s3_max_get_burst.changed) && settings.s3_max_get_rps)
|
|
|
|
get_request_throttler = std::make_shared<Throttler>(
|
|
|
|
settings.s3_max_get_rps, settings.s3_max_get_burst ? settings.s3_max_get_burst : Throttler::default_burst_seconds * settings.s3_max_get_rps);
|
|
|
|
|
|
|
|
if ((!if_changed || settings.s3_max_put_rps.changed || settings.s3_max_put_burst.changed) && settings.s3_max_put_rps)
|
|
|
|
put_request_throttler = std::make_shared<Throttler>(
|
|
|
|
settings.s3_max_put_rps, settings.s3_max_put_burst ? settings.s3_max_put_burst : Throttler::default_burst_seconds * settings.s3_max_put_rps);
|
2023-02-03 04:27:13 +00:00
|
|
|
|
2023-05-21 10:42:28 +00:00
|
|
|
if (!if_changed || settings.s3_throw_on_zero_files_match.changed)
|
2023-02-04 14:11:14 +00:00
|
|
|
throw_on_zero_files_match = settings.s3_throw_on_zero_files_match;
|
2023-05-21 10:42:28 +00:00
|
|
|
|
|
|
|
if (!if_changed || settings.s3_retry_attempts.changed)
|
|
|
|
retry_attempts = settings.s3_retry_attempts;
|
2023-06-19 23:35:35 +00:00
|
|
|
|
|
|
|
if (!if_changed || settings.s3_request_timeout_ms.changed)
|
|
|
|
request_timeout_ms = settings.s3_request_timeout_ms;
|
2022-12-17 17:12:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void S3Settings::RequestSettings::updateFromSettings(const Settings & settings)
|
|
|
|
{
|
|
|
|
updateFromSettingsImpl(settings, true);
|
|
|
|
upload_settings.updateFromSettings(settings);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config, const Settings & settings)
|
|
|
|
{
|
|
|
|
std::lock_guard lock(mutex);
|
|
|
|
s3_settings.clear();
|
|
|
|
if (!config.has(config_elem))
|
|
|
|
return;
|
2022-08-23 13:12:09 +00:00
|
|
|
|
2022-12-17 14:09:53 +00:00
|
|
|
Poco::Util::AbstractConfiguration::Keys config_keys;
|
|
|
|
config.keys(config_elem, config_keys);
|
2022-08-23 13:12:09 +00:00
|
|
|
|
2020-06-01 17:16:09 +00:00
|
|
|
for (const String & key : config_keys)
|
|
|
|
{
|
2021-01-07 03:42:39 +00:00
|
|
|
if (config.has(config_elem + "." + key + ".endpoint"))
|
2020-12-10 09:19:42 +00:00
|
|
|
{
|
2022-12-17 14:09:53 +00:00
|
|
|
auto endpoint = config.getString(config_elem + "." + key + ".endpoint");
|
2022-09-15 07:45:28 +00:00
|
|
|
auto auth_settings = S3::AuthSettings::loadFromConfig(config_elem + "." + key, config);
|
2022-12-17 14:09:53 +00:00
|
|
|
S3Settings::RequestSettings request_settings(config, config_elem + "." + key, settings);
|
2022-11-17 16:35:04 +00:00
|
|
|
|
|
|
|
s3_settings.emplace(endpoint, S3Settings{std::move(auth_settings), std::move(request_settings)});
|
2021-01-07 03:42:39 +00:00
|
|
|
}
|
2020-06-01 17:16:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-19 15:11:29 +00:00
|
|
|
S3Settings StorageS3Settings::getSettings(const String & endpoint, const String & user) const
|
2020-06-01 17:16:09 +00:00
|
|
|
{
|
2020-06-04 13:48:20 +00:00
|
|
|
std::lock_guard lock(mutex);
|
2022-04-03 22:33:59 +00:00
|
|
|
auto next_prefix_setting = s3_settings.upper_bound(endpoint);
|
2021-01-07 03:42:39 +00:00
|
|
|
|
|
|
|
/// Linear time algorithm may be replaced with logarithmic with prefix tree map.
|
2022-04-03 22:33:59 +00:00
|
|
|
for (auto possible_prefix_setting = next_prefix_setting; possible_prefix_setting != s3_settings.begin();)
|
2021-01-07 03:42:39 +00:00
|
|
|
{
|
|
|
|
std::advance(possible_prefix_setting, -1);
|
2024-02-19 15:11:29 +00:00
|
|
|
const auto & [endpoint_prefix, settings] = *possible_prefix_setting;
|
|
|
|
if (boost::algorithm::starts_with(endpoint, endpoint_prefix) && settings.auth_settings.canBeUsedByUser(user))
|
2021-01-07 03:42:39 +00:00
|
|
|
return possible_prefix_setting->second;
|
|
|
|
}
|
|
|
|
|
2020-06-01 17:16:09 +00:00
|
|
|
return {};
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|