From dec8efc39ea26e5417a4094b3be6d1e82ee48bcd Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Sun, 29 Oct 2023 02:04:38 +0000 Subject: [PATCH] make az as attribute of keepersnapshotmangers3. empty config not working. https://pastila.nl/?00048e4f/d56af90d13f2d871e30dfb1db9660266#yff6d8HVQm0blQUUiZg8SQ== Signed-off-by: Jianfei Hu --- src/Coordination/KeeperContext.cpp | 18 ++--- src/Coordination/KeeperContext.h | 2 +- src/Coordination/KeeperDispatcher.cpp | 4 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 2 + src/Coordination/KeeperSnapshotManagerS3.h | 6 ++ src/IO/S3/Credentials.cpp | 75 +++++++++++--------- src/IO/S3/Credentials.h | 3 + 7 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp index bc891a7a1f5..4ce8b2c732a 100644 --- a/src/Coordination/KeeperContext.cpp +++ b/src/Coordination/KeeperContext.cpp @@ -9,13 +9,6 @@ #include #include -#if USE_AWS_S3 - -#include -#include -#include -#endif - namespace DB { @@ -37,16 +30,15 @@ KeeperContext::KeeperContext(bool standalone_keeper_) /// for older clients, the default is equivalent to WITH_MULTI_READ version system_nodes_with_data[keeper_api_version_path] = toString(static_cast(KeeperApiVersion::WITH_MULTI_READ)); - - #if USE_AWS_S3 - auto metadata_client = S3::InitEC2MetadataClient(Aws::Client::ClientConfiguration{}); - running_availability_zone = metadata_client->getCurrentAvailabilityZone(); - #endif } -void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, KeeperDispatcher * dispatcher_) +void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, KeeperDispatcher * dispatcher_, std::string availability_zone) { dispatcher = dispatcher_; + + running_availability_zone = availability_zone; + LOG_INFO(&Poco::Logger::get("KeeperContext"), "Initialize the KeeperContext with availability zone: '{}'. ", running_availability_zone); + digest_enabled = config.getBool("keeper_server.digest_enabled", false); ignore_system_path_on_startup = config.getBool("keeper_server.ignore_system_path_on_startup", false); diff --git a/src/Coordination/KeeperContext.h b/src/Coordination/KeeperContext.h index ae55f0cd844..a7ef7d9e937 100644 --- a/src/Coordination/KeeperContext.h +++ b/src/Coordination/KeeperContext.h @@ -23,7 +23,7 @@ public: SHUTDOWN }; - void initialize(const Poco::Util::AbstractConfiguration & config, KeeperDispatcher * dispatcher_); + void initialize(const Poco::Util::AbstractConfiguration & config, KeeperDispatcher * dispatcher_, std::string availability_zone); Phase getServerState() const; void setServerState(Phase server_state_); diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index d93bdb21db0..5807ab8d188 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "IO/S3/Credentials.h" #include #include @@ -368,9 +369,8 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf snapshot_thread = ThreadFromGlobalPool([this] { snapshotThread(); }); snapshot_s3.startup(config, macros); - keeper_context = std::make_shared(standalone_keeper); - keeper_context->initialize(config, this); + keeper_context->initialize(config, this, snapshot_s3.getAvaibilityZone()); server = std::make_unique( configuration_and_settings, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 302e05c8418..783a8ed8c20 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -98,6 +98,8 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo client_configuration.endpointOverride = new_uri.endpoint; + avaibility_zone = S3::determineAvailabilityZone(client_configuration); + auto client = S3::ClientFactory::instance().create( client_configuration, new_uri.is_virtual_hosted_style, diff --git a/src/Coordination/KeeperSnapshotManagerS3.h b/src/Coordination/KeeperSnapshotManagerS3.h index e17cf5a1cfb..25684d4d054 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.h +++ b/src/Coordination/KeeperSnapshotManagerS3.h @@ -9,6 +9,7 @@ #include #if USE_AWS_S3 +#include #include #include @@ -29,6 +30,8 @@ public: void updateS3Configuration(const Poco::Util::AbstractConfiguration & config, const MultiVersion::Version & macros); void uploadSnapshot(const SnapshotFileInfo & file_info, bool async_upload = true); + std::string getAvaibilityZone() const { return avaibility_zone; } + /// 'macros' are used to substitute macros in endpoint of disks void startup(const Poco::Util::AbstractConfiguration & config, const MultiVersion::Version & macros); void shutdown(); @@ -50,6 +53,7 @@ private: UUID uuid; std::shared_ptr getSnapshotS3Client() const; + std::string avaibility_zone; void uploadSnapshotImpl(const SnapshotFileInfo & snapshot_file_info); @@ -65,6 +69,8 @@ public: void updateS3Configuration(const Poco::Util::AbstractConfiguration &, const MultiVersion::Version &) {} void uploadSnapshot(const SnapshotFileInfo &, [[maybe_unused]] bool async_upload = true) {} + std::string getAvaibilityZone() const { return ""; } + void startup(const Poco::Util::AbstractConfiguration &, const MultiVersion::Version &) {} void shutdown() {} diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index ccb1ff6f55b..15be25a82d7 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -584,6 +584,40 @@ Aws::String SSOCredentialsProvider::loadAccessTokenFile(const Aws::String & sso_ } } +Aws::Client::ClientConfiguration getAwsClientConfig(const DB::S3::PocoHTTPClientConfiguration & configuration) +{ + DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration( + configuration.region, + configuration.remote_host_filter, + configuration.s3_max_redirects, + configuration.s3_retry_attempts, + configuration.enable_s3_requests_logging, + configuration.for_disk_s3, + configuration.get_request_throttler, + configuration.put_request_throttler, + Aws::Http::SchemeMapper::ToString(Aws::Http::Scheme::HTTP)); + + /// See MakeDefaultHttpResourceClientConfiguration(). + /// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside + /// of contrib/aws/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp + aws_client_configuration.maxConnections = 2; + + /// Explicitly set the proxy settings to empty/zero to avoid relying on defaults that could potentially change + /// in the future. + aws_client_configuration.proxyHost = ""; + aws_client_configuration.proxyUserName = ""; + aws_client_configuration.proxyPassword = ""; + aws_client_configuration.proxyPort = 0; + + /// EC2MetadataService throttles by delaying the response so the service client should set a large read timeout. + /// EC2MetadataService delay is in order of seconds so it only make sense to retry after a couple of seconds. + aws_client_configuration.connectTimeoutMs = 1000; + aws_client_configuration.requestTimeoutMs = 1000; + + aws_client_configuration.retryStrategy = std::make_shared(1, 1000); + return aws_client_configuration; +} + S3CredentialsProviderChain::S3CredentialsProviderChain( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, @@ -674,38 +708,8 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( } else if (Aws::Utils::StringUtils::ToLower(ec2_metadata_disabled.c_str()) != "true") { - DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration( - configuration.region, - configuration.remote_host_filter, - configuration.s3_max_redirects, - configuration.s3_retry_attempts, - configuration.enable_s3_requests_logging, - configuration.for_disk_s3, - configuration.get_request_throttler, - configuration.put_request_throttler, - Aws::Http::SchemeMapper::ToString(Aws::Http::Scheme::HTTP)); - - /// See MakeDefaultHttpResourceClientConfiguration(). - /// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside - /// of contrib/aws/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp - aws_client_configuration.maxConnections = 2; - - /// Explicitly set the proxy settings to empty/zero to avoid relying on defaults that could potentially change - /// in the future. - aws_client_configuration.proxyHost = ""; - aws_client_configuration.proxyUserName = ""; - aws_client_configuration.proxyPassword = ""; - aws_client_configuration.proxyPort = 0; - - /// EC2MetadataService throttles by delaying the response so the service client should set a large read timeout. - /// EC2MetadataService delay is in order of seconds so it only make sense to retry after a couple of seconds. - aws_client_configuration.connectTimeoutMs = 1000; - aws_client_configuration.requestTimeoutMs = 1000; - - aws_client_configuration.retryStrategy = std::make_shared(1, 1000); - - // Code that we want to copy. - auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration); + auto aws_client_config = getAwsClientConfig(configuration); + auto ec2_metadata_client = InitEC2MetadataClient(aws_client_config); auto config_loader = std::make_shared(ec2_metadata_client, !credentials_configuration.use_insecure_imds_request); AddProvider(std::make_shared(config_loader)); @@ -718,8 +722,13 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( AddProvider(std::make_shared()); } +std::string determineAvailabilityZone(const DB::S3::PocoHTTPClientConfiguration & configuration) +{ + auto aws_client_configuration = getAwsClientConfig(configuration); + auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration); + return ec2_metadata_client->getCurrentAvailabilityZone(); } } - +} #endif diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 0243e8e4986..c81d06f9fd4 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -166,6 +166,9 @@ struct CredentialsConfiguration bool no_sign_request = false; }; + +std::string determineAvailabilityZone(const DB::S3::PocoHTTPClientConfiguration & configuration); + class S3CredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain { public: