thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 21:24:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Some refactoring.

Browse Source
This commit is contained in:
Vladimir Chebotarev 2021-03-31 10:26:30 +03:00
parent c0ea3ce08a
commit 116423b30c
1 changed files with 77 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
src/IO/S3Common.cpp
View File

@ -103,6 +103,8 @@ class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient
static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600"; static constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600";
static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds"; static constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "x-aws-ec2-metadata-token-ttl-seconds";
static constexpr char EC2_DEFAULT_METADATA_ENDPOINT[] = "http://169.254.169.254";
public: public:
/// See EC2MetadataClient. /// See EC2MetadataClient.
@ -130,35 +132,36 @@ public:
virtual Aws::String GetDefaultCredentials() const virtual Aws::String GetDefaultCredentials() const
{ {
std::unique_lock<std::recursive_mutex> locker(token_mutex); String credentials_string;
LOG_TRACE(logger, "Getting default credentials for ec2 instance.");
auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr);
Aws::String credentialsString = result.GetPayload();
auto httpResponseCode = result.GetResponseCode();
if (httpResponseCode == Aws::Http::HttpResponseCode::UNAUTHORIZED)
{ {
return {}; std::unique_lock<std::recursive_mutex> locker(token_mutex);
LOG_TRACE(logger, "Getting default credentials for EC2 instance.");
auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr);
credentials_string = result.GetPayload();
if (result.GetResponseCode() == Aws::Http::HttpResponseCode::UNAUTHORIZED)
{
return {};
}
} }
locker.unlock();
Aws::String trimmedCredentialsString = Aws::Utils::StringUtils::Trim(credentialsString.c_str()); String trimmed_credentials_string = Aws::Utils::StringUtils::Trim(credentials_string.c_str());
if (trimmedCredentialsString.empty()) return {}; if (trimmed_credentials_string.empty())
return {};
Aws::Vector<Aws::String> securityCredentials = Aws::Utils::StringUtils::Split(trimmedCredentialsString, '\n'); std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_credentials_string, '\n');
LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} returned credential string {}.", LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} returned credential string {}.",
EC2_SECURITY_CREDENTIALS_RESOURCE, trimmedCredentialsString); EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_credentials_string);
if (securityCredentials.size() == 0) if (security_credentials.size() == 0)
{ {
LOG_WARNING(logger, "Initial call to ec2Metadataservice to get credentials failed."); LOG_WARNING(logger, "Initial call to EC2MetadataService to get credentials failed.");
return {}; return {};
} }
Aws::StringStream ss; Aws::StringStream ss;
ss << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << securityCredentials[0]; ss << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
LOG_DEBUG(logger, "Calling EC2MetadataService resource {}.", ss.str()); LOG_DEBUG(logger, "Calling EC2MetadataService resource {}.", ss.str());
return GetResource(ss.str().c_str()); return GetResource(ss.str().c_str());
} }
@ -173,59 +176,64 @@ public:
virtual Aws::String GetDefaultCredentialsSecurely() const virtual Aws::String GetDefaultCredentialsSecurely() const
{ {
std::unique_lock<std::recursive_mutex> locker(token_mutex); String user_agent_string = AWSComputeUserAgentString();
String new_token;
Aws::StringStream ss; {
ss << endpoint << EC2_IMDS_TOKEN_RESOURCE; std::unique_lock<std::recursive_mutex> locker(token_mutex);
std::shared_ptr<Aws::Http::HttpRequest> tokenRequest(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_PUT,
Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
tokenRequest->SetHeaderValue(EC2_IMDS_TOKEN_TTL_HEADER, EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE);
auto userAgentString = AWSComputeUserAgentString();
tokenRequest->SetUserAgent(userAgentString);
LOG_TRACE(logger, "Calling EC2MetadataService to get token.");
auto result = GetResourceWithAWSWebServiceResult(tokenRequest);
Aws::String tokenString = result.GetPayload();
Aws::String trimmedTokenString = Aws::Utils::StringUtils::Trim(tokenString.c_str());
if (result.GetResponseCode() == Aws::Http::HttpResponseCode::BAD_REQUEST) Aws::StringStream ss;
{ ss << endpoint << EC2_IMDS_TOKEN_RESOURCE;
return {}; std::shared_ptr<Aws::Http::HttpRequest> token_request(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_PUT,
}
else if (result.GetResponseCode() != Aws::Http::HttpResponseCode::OK || trimmedTokenString.empty())
{
LOG_TRACE(logger, "Calling EC2MetadataService to get token failed, falling back to less secure way.");
return GetDefaultCredentials();
}
token = trimmedTokenString;
locker.unlock();
ss.str("");
ss << endpoint << EC2_SECURITY_CREDENTIALS_RESOURCE;
std::shared_ptr<Aws::Http::HttpRequest> profileRequest(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_GET,
Aws::Utils::Stream::DefaultResponseStreamFactoryMethod)); Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
profileRequest->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, trimmedTokenString); token_request->SetHeaderValue(EC2_IMDS_TOKEN_TTL_HEADER, EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE);
profileRequest->SetUserAgent(userAgentString); token_request->SetUserAgent(user_agent_string);
Aws::String profileString = GetResourceWithAWSWebServiceResult(profileRequest).GetPayload(); LOG_TRACE(logger, "Calling EC2MetadataService to get token.");
auto result = GetResourceWithAWSWebServiceResult(token_request);
String token_string = result.GetPayload();
new_token = Aws::Utils::StringUtils::Trim(token_string.c_str());
Aws::String trimmedProfileString = Aws::Utils::StringUtils::Trim(profileString.c_str()); if (result.GetResponseCode() == Aws::Http::HttpResponseCode::BAD_REQUEST)
Aws::Vector<Aws::String> securityCredentials = Aws::Utils::StringUtils::Split(trimmedProfileString, '\n'); {
return {};
}
else if (result.GetResponseCode() != Aws::Http::HttpResponseCode::OK || new_token.empty())
{
LOG_TRACE(logger, "Calling EC2MetadataService to get token failed, falling back to less secure way.");
return GetDefaultCredentials();
}
token = new_token;
}
String url = endpoint + EC2_SECURITY_CREDENTIALS_RESOURCE;
std::shared_ptr<Aws::Http::HttpRequest> profile_request(Aws::Http::CreateHttpRequest(url,
Aws::Http::HttpMethod::HTTP_GET,
Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
profile_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
profile_request->SetUserAgent(user_agent_string);
String profile_string = GetResourceWithAWSWebServiceResult(profile_request).GetPayload();
String trimmed_profile_string = Aws::Utils::StringUtils::Trim(profile_string.c_str());
std::vector<String> security_credentials = Aws::Utils::StringUtils::Split(trimmed_profile_string, '\n');
LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} with token returned profile string {}.", LOG_DEBUG(logger, "Calling EC2MetadataService resource, {} with token returned profile string {}.",
EC2_SECURITY_CREDENTIALS_RESOURCE, trimmedProfileString); EC2_SECURITY_CREDENTIALS_RESOURCE, trimmed_profile_string);
if (securityCredentials.size() == 0) if (security_credentials.size() == 0)
{ {
LOG_WARNING(logger, "Calling EC2Metadataservice to get profiles failed."); LOG_WARNING(logger, "Calling EC2Metadataservice to get profiles failed.");
return {}; return {};
} }
ss.str(""); Aws::StringStream ss;
ss << endpoint << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << securityCredentials[0]; ss << endpoint << EC2_SECURITY_CREDENTIALS_RESOURCE << "/" << security_credentials[0];
std::shared_ptr<Aws::Http::HttpRequest> credentialsRequest(Aws::Http::CreateHttpRequest(ss.str(), Aws::Http::HttpMethod::HTTP_GET, std::shared_ptr<Aws::Http::HttpRequest> credentials_request(Aws::Http::CreateHttpRequest(ss.str(),
Aws::Utils::Stream::DefaultResponseStreamFactoryMethod)); Aws::Http::HttpMethod::HTTP_GET,
credentialsRequest->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, trimmedTokenString); Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
credentialsRequest->SetUserAgent(userAgentString); credentials_request->SetHeaderValue(EC2_IMDS_TOKEN_HEADER, new_token);
credentials_request->SetUserAgent(user_agent_string);
LOG_DEBUG(logger, "Calling EC2MetadataService resource {} with token.", ss.str()); LOG_DEBUG(logger, "Calling EC2MetadataService resource {} with token.", ss.str());
return GetResourceWithAWSWebServiceResult(credentialsRequest).GetPayload(); return GetResourceWithAWSWebServiceResult(credentials_request).GetPayload();
} }
virtual Aws::String GetCurrentRegion() const virtual Aws::String GetCurrentRegion() const
@ -234,10 +242,9 @@ public:
} }
private: private:
Aws::String endpoint = "http://169.254.169.254"; const Aws::String endpoint = EC2_DEFAULT_METADATA_ENDPOINT;
mutable std::recursive_mutex token_mutex; mutable std::recursive_mutex token_mutex;
mutable Aws::String token; mutable Aws::String token;
mutable Aws::String region;
Poco::Logger * logger; Poco::Logger * logger;
}; };
@ -261,27 +268,28 @@ protected:
auto credentials_str = use_secure_pull ? client->GetDefaultCredentialsSecurely() : client->GetDefaultCredentials(); auto credentials_str = use_secure_pull ? client->GetDefaultCredentialsSecurely() : client->GetDefaultCredentials();
/// See EC2InstanceProfileConfigLoader. /// See EC2InstanceProfileConfigLoader.
if (credentials_str.empty()) return false; if (credentials_str.empty())
return false;
Aws::Utils::Json::JsonValue credentialsDoc(credentials_str); Aws::Utils::Json::JsonValue credentials_doc(credentials_str);
if (!credentialsDoc.WasParseSuccessful()) if (!credentials_doc.WasParseSuccessful())
{ {
LOG_ERROR(logger, "Failed to parse output from EC2MetadataService."); LOG_ERROR(logger, "Failed to parse output from EC2MetadataService.");
return false; return false;
} }
Aws::String accessKey, secretKey, token; String access_key, secret_key, token;
auto credentialsView = credentialsDoc.View(); auto credentials_view = credentials_doc.View();
accessKey = credentialsView.GetString("AccessKeyId"); access_key = credentials_view.GetString("AccessKeyId");
LOG_ERROR(logger, "Successfully pulled credentials from metadata service with access key {}.", accessKey); LOG_ERROR(logger, "Successfully pulled credentials from EC2MetadataService with access key {}.", access_key);
secretKey = credentialsView.GetString("SecretAccessKey"); secret_key = credentials_view.GetString("SecretAccessKey");
token = credentialsView.GetString("Token"); token = credentials_view.GetString("Token");
auto region = client->GetCurrentRegion(); auto region = client->GetCurrentRegion();
Aws::Config::Profile profile; Aws::Config::Profile profile;
profile.SetCredentials(Aws::Auth::AWSCredentials(accessKey, secretKey, token)); profile.SetCredentials(Aws::Auth::AWSCredentials(access_key, secret_key, token));
profile.SetRegion(region); profile.SetRegion(region);
profile.SetName(Aws::Config::INSTANCE_PROFILE_KEY); profile.SetName(Aws::Config::INSTANCE_PROFILE_KEY);