From 0a19b4fbd68f1c6673146a8e76f3406f62b15d68 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 5 Nov 2019 10:54:13 +0300 Subject: [PATCH] Attempt to add S3 authentication. --- dbms/src/IO/ReadBufferFromS3.cpp | 7 +++-- dbms/src/IO/ReadBufferFromS3.h | 3 --- dbms/src/IO/S3Common.cpp | 44 +++++++++++++++++++++++++++++++ dbms/src/IO/S3Common.h | 18 +++++++++++++ dbms/src/IO/WriteBufferFromS3.cpp | 7 +++++ 5 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 dbms/src/IO/S3Common.cpp create mode 100644 dbms/src/IO/S3Common.h diff --git a/dbms/src/IO/ReadBufferFromS3.cpp b/dbms/src/IO/ReadBufferFromS3.cpp index 718f9d21f8a..b26a8b8c316 100644 --- a/dbms/src/IO/ReadBufferFromS3.cpp +++ b/dbms/src/IO/ReadBufferFromS3.cpp @@ -1,6 +1,7 @@ #include #include +#include #include @@ -16,12 +17,8 @@ ReadBufferFromS3::ReadBufferFromS3(const Poco::URI & uri_, const ConnectionTimeouts & timeouts) : ReadBuffer(nullptr, 0) , uri {uri_} - , access_key_id {access_key_id_} - , secret_access_key {secret_access_key_} , session {makeHTTPSession(uri_, timeouts)} { - /// FIXME: Implement rest of S3 authorization. - Poco::Net::HTTPResponse response; std::unique_ptr request; @@ -37,6 +34,8 @@ ReadBufferFromS3::ReadBufferFromS3(const Poco::URI & uri_, Poco::Net::HTTPRequest::HTTP_1_1); request->setHost(uri.getHost()); // use original, not resolved host name in header + S3Helper::authenticateRequest(*request, access_key_id_, secret_access_key_); + LOG_TRACE((&Logger::get("ReadBufferFromS3")), "Sending request to " << uri.toString()); session->sendRequest(*request); diff --git a/dbms/src/IO/ReadBufferFromS3.h b/dbms/src/IO/ReadBufferFromS3.h index e787828f0e7..071ee7802a2 100644 --- a/dbms/src/IO/ReadBufferFromS3.h +++ b/dbms/src/IO/ReadBufferFromS3.h @@ -17,9 +17,6 @@ class ReadBufferFromS3 : public ReadBuffer { protected: Poco::URI uri; - String access_key_id; - String secret_access_key; - HTTPSessionPtr session; std::istream * istr; /// owned by session std::unique_ptr impl; diff --git a/dbms/src/IO/S3Common.cpp b/dbms/src/IO/S3Common.cpp new file mode 100644 index 00000000000..dea2f888662 --- /dev/null +++ b/dbms/src/IO/S3Common.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ + +void S3Helper::authenticateRequest(Poco::Net::HTTPRequest & request, + const String & access_key_id, + const String & secret_access_key) +{ + /// See https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html + + if (access_key_id.empty()) + return; + + /// Limitations: + /// 1. Virtual hosted-style requests are not supported. + /// 2. AMZ headers are not supported (TODO). + + String string_to_sign = request.getMethod() + "\n" + + request.get("Content-MD5", "") + "\n" + + request.get("Content-Type", "") + "\n" + + request.get("Date", "") + "\n" + + Poco::URI(request.getURI()).getPath(); + + Poco::HMACEngine engine(secret_access_key); + engine.update(string_to_sign); + auto digest = engine.digest(); + std::ostringstream signature; + Poco::Base64Encoder encoder(signature); + std::copy(digest.begin(), digest.end(), std::ostream_iterator(encoder)); + + request.set("Authorization", "AWS " + access_key_id + ":" + signature.str()); +} + +} \ No newline at end of file diff --git a/dbms/src/IO/S3Common.h b/dbms/src/IO/S3Common.h new file mode 100644 index 00000000000..2f18021c93e --- /dev/null +++ b/dbms/src/IO/S3Common.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + + +namespace DB +{ + +struct S3Helper +{ + static void authenticateRequest(Poco::Net::HTTPRequest & request, + const String & access_key_id, + const String & secret_access_key); +}; + +} diff --git a/dbms/src/IO/WriteBufferFromS3.cpp b/dbms/src/IO/WriteBufferFromS3.cpp index 977a5b22fdc..4154db48282 100644 --- a/dbms/src/IO/WriteBufferFromS3.cpp +++ b/dbms/src/IO/WriteBufferFromS3.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -113,6 +114,8 @@ void WriteBufferFromS3::initiate() request_ptr = std::make_unique(Poco::Net::HTTPRequest::HTTP_POST, initiate_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); request_ptr->setHost(initiate_uri.getHost()); // use original, not resolved host name in header + S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key); + request_ptr->setContentLength(0); LOG_TRACE((&Logger::get("WriteBufferFromS3")), "Sending request to " << initiate_uri.toString()); @@ -173,6 +176,8 @@ void WriteBufferFromS3::writePart(const String & data) request_ptr = std::make_unique(Poco::Net::HTTPRequest::HTTP_PUT, part_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); request_ptr->setHost(part_uri.getHost()); // use original, not resolved host name in header + S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key); + request_ptr->setExpectContinue(true); request_ptr->setContentLength(data.size()); @@ -240,6 +245,8 @@ void WriteBufferFromS3::complete() request_ptr = std::make_unique(Poco::Net::HTTPRequest::HTTP_POST, complete_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); request_ptr->setHost(complete_uri.getHost()); // use original, not resolved host name in header + S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key); + request_ptr->setExpectContinue(true); request_ptr->setContentLength(data.size());