thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-04 13:32:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

sign all aws headers

Browse Source
This commit is contained in:
Arthur Passos 2023-11-20 13:38:32 -03:00
parent 2a59a688de
commit e5129990ed
4 changed files with 70 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/IO/S3/Client.cpp
View File

@ -103,6 +103,20 @@ void verifyClientConfiguration(const Aws::Client::ClientConfiguration & client_c
assert_cast<const Client::RetryStrategy &>(*client_config.retryStrategy);
}
void addAdditionalAMZHeadersToCanonicalHeadersList(
Aws::AmazonWebServiceRequest & request,
const HTTPHeaderEntries & extra_headers
)
{
for (const auto & [name, value] : extra_headers)
{
if (name.starts_with("x-amz-"))
{
request.SetAdditionalCustomHeaderValue(name, value);
}
}
}
}
std::unique_ptr<Client> Client::create(
@ -265,12 +279,14 @@ template void Client::setKMSHeaders<CreateMultipartUploadRequest>(CreateMultipar
template void Client::setKMSHeaders<CopyObjectRequest>(CopyObjectRequest & request) const;
template void Client::setKMSHeaders<PutObjectRequest>(PutObjectRequest & request) const;
Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) const
Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const
{
const auto & bucket = request.GetBucket();
request.setApiMode(api_mode);
addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers);
if (auto region = getRegionForBucket(bucket); !region.empty())
{
if (!detect_region)
@ -346,36 +362,36 @@ Model::HeadObjectOutcome Client::HeadObject(const HeadObjectRequest & request) c
/// For each request, we wrap the request functions from Aws::S3::Client with doRequest
/// doRequest calls virtuall function from Aws::S3::Client while DB::S3::Client has not virtual calls for each request type
Model::ListObjectsV2Outcome Client::ListObjectsV2(const ListObjectsV2Request & request) const
Model::ListObjectsV2Outcome Client::ListObjectsV2(ListObjectsV2Request & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ true>(
request, [this](const Model::ListObjectsV2Request & req) { return ListObjectsV2(req); });
}
Model::ListObjectsOutcome Client::ListObjects(const ListObjectsRequest & request) const
Model::ListObjectsOutcome Client::ListObjects(ListObjectsRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ true>(
request, [this](const Model::ListObjectsRequest & req) { return ListObjects(req); });
}
Model::GetObjectOutcome Client::GetObject(const GetObjectRequest & request) const
Model::GetObjectOutcome Client::GetObject(GetObjectRequest & request) const
{
return doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); });
}
Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(const AbortMultipartUploadRequest & request) const
Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(AbortMultipartUploadRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::AbortMultipartUploadRequest & req) { return AbortMultipartUpload(req); });
}
Model::CreateMultipartUploadOutcome Client::CreateMultipartUpload(const CreateMultipartUploadRequest & request) const
Model::CreateMultipartUploadOutcome Client::CreateMultipartUpload(CreateMultipartUploadRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::CreateMultipartUploadRequest & req) { return CreateMultipartUpload(req); });
}
Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(const CompleteMultipartUploadRequest & request) const
Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(CompleteMultipartUploadRequest & request) const
{
auto outcome = doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::CompleteMultipartUploadRequest & req) { return CompleteMultipartUpload(req); });
@ -422,31 +438,31 @@ Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(const Comp
return outcome;
}
Model::CopyObjectOutcome Client::CopyObject(const CopyObjectRequest & request) const
Model::CopyObjectOutcome Client::CopyObject(CopyObjectRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::CopyObjectRequest & req) { return CopyObject(req); });
}
Model::PutObjectOutcome Client::PutObject(const PutObjectRequest & request) const
Model::PutObjectOutcome Client::PutObject(PutObjectRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::PutObjectRequest & req) { return PutObject(req); });
}
Model::UploadPartOutcome Client::UploadPart(const UploadPartRequest & request) const
Model::UploadPartOutcome Client::UploadPart(UploadPartRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::UploadPartRequest & req) { return UploadPart(req); });
}
Model::UploadPartCopyOutcome Client::UploadPartCopy(const UploadPartCopyRequest & request) const
Model::UploadPartCopyOutcome Client::UploadPartCopy(UploadPartCopyRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::UploadPartCopyRequest & req) { return UploadPartCopy(req); });
}
Model::DeleteObjectOutcome Client::DeleteObject(const DeleteObjectRequest & request) const
Model::DeleteObjectOutcome Client::DeleteObject(DeleteObjectRequest & request) const
{
return doRequestWithRetryNetworkErrors</*IsReadMethod*/ false>(
request, [this](const Model::DeleteObjectRequest & req) { return DeleteObject(req); });
@ -458,7 +474,7 @@ Model::DeleteObjectsOutcome Client::DeleteObjects(const DeleteObjectsRequest & r
request, [this](const Model::DeleteObjectsRequest & req) { return DeleteObjects(req); });
}
Client::ComposeObjectOutcome Client::ComposeObject(const ComposeObjectRequest & request) const
Client::ComposeObjectOutcome Client::ComposeObject(ComposeObjectRequest & request) const
{
auto request_fn = [this](const ComposeObjectRequest & req)
{
@ -490,8 +506,9 @@ Client::ComposeObjectOutcome Client::ComposeObject(const ComposeObjectRequest &
template <typename RequestType, typename RequestFn>
std::invoke_result_t<RequestFn, RequestType>
Client::doRequest(const RequestType & request, RequestFn request_fn) const
Client::doRequest(RequestType & request, RequestFn request_fn) const
{
addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers);
const auto & bucket = request.GetBucket();
request.setApiMode(api_mode);
@ -568,8 +585,9 @@ Client::doRequest(const RequestType & request, RequestFn request_fn) const
template <bool IsReadMethod, typename RequestType, typename RequestFn>
std::invoke_result_t<RequestFn, RequestType>
Client::doRequestWithRetryNetworkErrors(const RequestType & request, RequestFn request_fn) const
Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request_fn) const
{
addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers);
auto with_retries = [this, request_fn_ = std::move(request_fn)] (const RequestType & request_)
{
chassert(client_configuration.retryStrategy);
@ -666,6 +684,8 @@ std::string Client::getRegionForBucket(const std::string & bucket, bool force_de
Aws::S3::Model::HeadBucketRequest req;
req.SetBucket(bucket);
addAdditionalAMZHeadersToCanonicalHeadersList(req, client_configuration.extra_headers);
std::string region;
auto outcome = HeadBucket(req);
if (outcome.IsSuccess())

32
src/IO/S3/Client.h
View File

@ -177,24 +177,24 @@ public:
template <typename RequestType>
void setKMSHeaders(RequestType & request) const;
Model::HeadObjectOutcome HeadObject(const HeadObjectRequest & request) const;
Model::ListObjectsV2Outcome ListObjectsV2(const ListObjectsV2Request & request) const;
Model::ListObjectsOutcome ListObjects(const ListObjectsRequest & request) const;
Model::GetObjectOutcome GetObject(const GetObjectRequest & request) const;
Model::HeadObjectOutcome HeadObject(HeadObjectRequest & request) const;
Model::ListObjectsV2Outcome ListObjectsV2(ListObjectsV2Request & request) const;
Model::ListObjectsOutcome ListObjects(ListObjectsRequest & request) const;
Model::GetObjectOutcome GetObject(GetObjectRequest & request) const;
Model::AbortMultipartUploadOutcome AbortMultipartUpload(const AbortMultipartUploadRequest & request) const;
Model::CreateMultipartUploadOutcome CreateMultipartUpload(const CreateMultipartUploadRequest & request) const;
Model::CompleteMultipartUploadOutcome CompleteMultipartUpload(const CompleteMultipartUploadRequest & request) const;
Model::UploadPartOutcome UploadPart(const UploadPartRequest & request) const;
Model::UploadPartCopyOutcome UploadPartCopy(const UploadPartCopyRequest & request) const;
Model::AbortMultipartUploadOutcome AbortMultipartUpload(AbortMultipartUploadRequest & request) const;
Model::CreateMultipartUploadOutcome CreateMultipartUpload(CreateMultipartUploadRequest & request) const;
Model::CompleteMultipartUploadOutcome CompleteMultipartUpload(CompleteMultipartUploadRequest & request) const;
Model::UploadPartOutcome UploadPart(UploadPartRequest & request) const;
Model::UploadPartCopyOutcome UploadPartCopy(UploadPartCopyRequest & request) const;
Model::CopyObjectOutcome CopyObject(const CopyObjectRequest & request) const;
Model::PutObjectOutcome PutObject(const PutObjectRequest & request) const;
Model::DeleteObjectOutcome DeleteObject(const DeleteObjectRequest & request) const;
Model::DeleteObjectsOutcome DeleteObjects(const DeleteObjectsRequest & request) const;
Model::CopyObjectOutcome CopyObject(CopyObjectRequest & request) const;
Model::PutObjectOutcome PutObject(PutObjectRequest & request) const;
Model::DeleteObjectOutcome DeleteObject(DeleteObjectRequest & request) const;
Model::DeleteObjectsOutcome DeleteObjects(DeleteObjectsRequest & request) const;
using ComposeObjectOutcome = Aws::Utils::Outcome<Aws::NoResult, Aws::S3::S3Error>;
ComposeObjectOutcome ComposeObject(const ComposeObjectRequest & request) const;
ComposeObjectOutcome ComposeObject(ComposeObjectRequest & request) const;
using Aws::S3::S3Client::EnableRequestProcessing;
using Aws::S3::S3Client::DisableRequestProcessing;
@ -236,11 +236,11 @@ private:
template <typename RequestType, typename RequestFn>
std::invoke_result_t<RequestFn, RequestType>
doRequest(const RequestType & request, RequestFn request_fn) const;
doRequest(RequestType & request, RequestFn request_fn) const;
template <bool IsReadMethod, typename RequestType, typename RequestFn>
std::invoke_result_t<RequestFn, RequestType>
doRequestWithRetryNetworkErrors(const RequestType & request, RequestFn request_fn) const;
doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request_fn) const;
void updateURIForBucket(const std::string & bucket, S3::URI new_uri) const;
std::optional<S3::URI> getURIFromError(const Aws::S3::S3Error & error) const;

12
src/IO/S3/PocoHTTPClient.cpp
View File

@ -469,7 +469,17 @@ void PocoHTTPClient::makeRequestInternalImpl(
for (const auto & [header_name, header_value] : request.GetHeaders())
poco_request.set(header_name, header_value);
for (const auto & [header_name, header_value] : extra_headers)
poco_request.set(boost::algorithm::to_lower_copy(header_name), header_value);
{
// AWS S3 canonical headers must include `Host`, `Content-Type` and any `x-amz-*`.
// These headers will be signed. Custom S3 headers specified in ClickHouse storage conf are added in `extra_headers`.
// At this point in the stack trace, request has already been signed and any `x-amz-*` extra headers was already added
// to the canonical headers list. Therefore, we should not add them again to the request.
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
if (!header_name.starts_with("x-amz-"))
{
poco_request.set(boost::algorithm::to_lower_copy(header_name), header_value);
}
}
Poco::Net::HTTPResponse poco_response;

10
src/IO/S3/tests/gtest_aws_s3_client.cpp
View File

@ -175,7 +175,10 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead)
"host;"
"x-amz-api-version;"
"x-amz-content-sha256;"
"x-amz-date, ...\n"
"x-amz-date;"
"x-amz-server-side-encryption-customer-algorithm;"
"x-amz-server-side-encryption-customer-key;"
"x-amz-server-side-encryption-customer-key-md5, ...\n"
"x-amz-server-side-encryption-customer-algorithm: AES256\n"
"x-amz-server-side-encryption-customer-key: Kv/gDqdWVGIT4iDqg+btQvV3lc1idlm4WI+MMOyHOAw=\n"
"x-amz-server-side-encryption-customer-key-md5: fMNuOw6OLU5GG2vc6RTA+g==\n");
@ -196,7 +199,10 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersWrite)
"content-type;"
"host;"
"x-amz-content-sha256;"
"x-amz-date, ...\n"
"x-amz-date;"
"x-amz-server-side-encryption-customer-algorithm;"
"x-amz-server-side-encryption-customer-key;"
"x-amz-server-side-encryption-customer-key-md5, ...\n"
"x-amz-server-side-encryption-customer-algorithm: AES256\n"
"x-amz-server-side-encryption-customer-key: Kv/gDqdWVGIT4iDqg+btQvV3lc1idlm4WI+MMOyHOAw=\n"
"x-amz-server-side-encryption-customer-key-md5: fMNuOw6OLU5GG2vc6RTA+g==\n");