thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 21:24:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added region parameter for S3 storage and disk.

Browse Source
This commit is contained in:
Vladimir Chebotarev 2021-05-01 20:41:16 +03:00
parent 2514233e61
commit a6ab040b07
8 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Disks/S3/registerDiskS3.cpp
View File

@ -115,6 +115,7 @@ std::shared_ptr<Aws::S3::S3Client>
getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextConstPtr context)
{
S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
config.getString(config_prefix + ".region"),
context->getRemoteHostFilter(), context->getGlobalContext()->getSettingsRef().s3_max_redirects);
S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint")));

22
src/IO/S3/PocoHTTPClient.cpp
View File

@ -47,9 +47,11 @@ namespace DB::S3
{
PocoHTTPClientConfiguration::PocoHTTPClientConfiguration(
const String & force_region_,
const RemoteHostFilter & remote_host_filter_,
unsigned int s3_max_redirects_)
: remote_host_filter(remote_host_filter_)
: force_region(force_region_)
, remote_host_filter(remote_host_filter_)
, s3_max_redirects(s3_max_redirects_)
{
}
@ -63,15 +65,23 @@ void PocoHTTPClientConfiguration::updateSchemeAndRegion()
if (uri.getScheme() == "http")
scheme = Aws::Http::Scheme::HTTP;
String matched_region;
if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region))
if (force_region.empty())
{
boost::algorithm::to_lower(matched_region);
region = matched_region;
String matched_region;
if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region))
{
boost::algorithm::to_lower(matched_region);
region = matched_region;
}
else
{
/// In global mode AWS C++ SDK send `us-east-1` but accept switching to another one if being suggested.
region = Aws::Region::AWS_GLOBAL;
}
}
else
{
region = Aws::Region::AWS_GLOBAL;
region = force_region;
}
}
}

3
src/IO/S3/PocoHTTPClient.h
View File

@ -29,13 +29,14 @@ class ClientFactory;
struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration
{
String force_region;
const RemoteHostFilter & remote_host_filter;
unsigned int s3_max_redirects;
void updateSchemeAndRegion();
private:
PocoHTTPClientConfiguration(const RemoteHostFilter & remote_host_filter_, unsigned int s3_max_redirects_);
PocoHTTPClientConfiguration(const String & force_region_, const RemoteHostFilter & remote_host_filter_, unsigned int s3_max_redirects_);
/// Constructor of Aws::Client::ClientConfiguration must be called after AWS SDK initialization.
friend ClientFactory;

5
src/IO/S3Common.cpp
View File

@ -410,7 +410,7 @@ public:
}
else if (Aws::Utils::StringUtils::ToLower(ec2_metadata_disabled.c_str()) != "true")
{
DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(configuration.remote_host_filter, configuration.s3_max_redirects);
DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(configuration.region, configuration.remote_host_filter, configuration.s3_max_redirects);
/// See MakeDefaultHttpResourceClientConfiguration().
/// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside
@ -590,10 +590,11 @@ namespace S3
}
PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT
const String & force_region,
const RemoteHostFilter & remote_host_filter,
unsigned int s3_max_redirects)
{
return PocoHTTPClientConfiguration(remote_host_filter, s3_max_redirects);
return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects);
}
URI::URI(const Poco::URI & uri_)

1
src/IO/S3Common.h
View File

@ -42,6 +42,7 @@ public:
bool use_insecure_imds_request);
PocoHTTPClientConfiguration createClientConfiguration(
const String & force_region,
const RemoteHostFilter & remote_host_filter,
unsigned int s3_max_redirects);

1
src/Storages/StorageS3.cpp
View File

@ -444,6 +444,7 @@ void StorageS3::updateClientAndAuthSettings(ContextPtr ctx, StorageS3::ClientAut
}
S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
settings.region,
ctx->getRemoteHostFilter(), ctx->getGlobalContext()->getSettingsRef().s3_max_redirects);
client_configuration.endpointOverride = upd.uri.endpoint;

10
src/Storages/StorageS3Settings.cpp
View File

@ -30,6 +30,7 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
auto endpoint = config.getString(config_elem + "." + key + ".endpoint");
auto access_key_id = config.getString(config_elem + "." + key + ".access_key_id", "");
auto secret_access_key = config.getString(config_elem + "." + key + ".secret_access_key", "");
auto region = config.getString(config_elem + "." + key + ".region", "");
auto server_side_encryption_customer_key_base64 = config.getString(config_elem + "." + key + ".server_side_encryption_customer_key_base64", "");
std::optional<bool> use_environment_credentials;
if (config.has(config_elem + "." + key + ".use_environment_credentials"))
@ -57,7 +58,14 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
}
}
settings.emplace(endpoint, S3AuthSettings{std::move(access_key_id), std::move(secret_access_key), std::move(server_side_encryption_customer_key_base64), std::move(headers), use_environment_credentials, use_insecure_imds_request});
settings.emplace(endpoint, S3AuthSettings{
std::move(access_key_id), std::move(secret_access_key),
std::move(region),
std::move(server_side_encryption_customer_key_base64),
std::move(headers),
use_environment_credentials,
use_insecure_imds_request
});
}
}
}

5
src/Storages/StorageS3Settings.h
View File

@ -28,6 +28,7 @@ struct S3AuthSettings
{
String access_key_id;
String secret_access_key;
String region;
String server_side_encryption_customer_key_base64;
HeaderCollection headers;
@ -38,7 +39,9 @@ struct S3AuthSettings
inline bool operator==(const S3AuthSettings & other) const
{
return access_key_id == other.access_key_id && secret_access_key == other.secret_access_key
&& server_side_encryption_customer_key_base64 == other.server_side_encryption_customer_key_base64 && headers == other.headers
&& region == other.region
&& server_side_encryption_customer_key_base64 == other.server_side_encryption_customer_key_base64
&& headers == other.headers
&& use_environment_credentials == other.use_environment_credentials
&& use_insecure_imds_request == other.use_insecure_imds_request;
}