Attempt to add S3 authentication.

dbms/src/IO/ReadBufferFromS3.cpp

@@ -1,6 +1,7 @@
 #include <IO/ReadBufferFromS3.h>
 
 #include <IO/ReadBufferFromIStream.h>
+#include <IO/S3Common.h>
 
 #include <common/logger_useful.h>
 
@@ -16,12 +17,8 @@ ReadBufferFromS3::ReadBufferFromS3(const Poco::URI & uri_,
     const ConnectionTimeouts & timeouts)
     : ReadBuffer(nullptr, 0)
     , uri {uri_}
-    , access_key_id {access_key_id_}
-    , secret_access_key {secret_access_key_}
     , session {makeHTTPSession(uri_, timeouts)}
 {
-    /// FIXME: Implement rest of S3 authorization.
-
     Poco::Net::HTTPResponse response;
     std::unique_ptr<Poco::Net::HTTPRequest> request;
 
@@ -37,6 +34,8 @@ ReadBufferFromS3::ReadBufferFromS3(const Poco::URI & uri_,
             Poco::Net::HTTPRequest::HTTP_1_1);
         request->setHost(uri.getHost()); // use original, not resolved host name in header
 
+        S3Helper::authenticateRequest(*request, access_key_id_, secret_access_key_);
+
         LOG_TRACE((&Logger::get("ReadBufferFromS3")), "Sending request to " << uri.toString());
 
         session->sendRequest(*request);

dbms/src/IO/ReadBufferFromS3.h

@@ -17,9 +17,6 @@ class ReadBufferFromS3 : public ReadBuffer
 {
 protected:
     Poco::URI uri;
-    String access_key_id;
-    String secret_access_key;
-
     HTTPSessionPtr session;
     std::istream * istr; /// owned by session
     std::unique_ptr<ReadBuffer> impl;

dbms/src/IO/S3Common.cpp

@@ -0,0 +1,44 @@
+#include <IO/S3Common.h>
+
+#include <iterator>
+#include <sstream>
+
+#include <Poco/Base64Encoder.h>
+#include <Poco/HMACEngine.h>
+#include <Poco/SHA1Engine.h>
+#include <Poco/URI.h>
+
+
+namespace DB
+{
+
+void S3Helper::authenticateRequest(Poco::Net::HTTPRequest & request,
+    const String & access_key_id,
+    const String & secret_access_key)
+{
+    /// See https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html
+
+    if (access_key_id.empty())
+        return;
+
+    /// Limitations:
+    /// 1. Virtual hosted-style requests are not supported.
+    /// 2. AMZ headers are not supported (TODO).
+
+    String string_to_sign = request.getMethod() + "\n"
+        + request.get("Content-MD5", "") + "\n"
+        + request.get("Content-Type", "") + "\n"
+        + request.get("Date", "") + "\n"
+        + Poco::URI(request.getURI()).getPath();
+
+    Poco::HMACEngine<Poco::SHA1Engine> engine(secret_access_key);
+    engine.update(string_to_sign);
+    auto digest = engine.digest();
+    std::ostringstream signature;
+    Poco::Base64Encoder encoder(signature);
+    std::copy(digest.begin(), digest.end(), std::ostream_iterator<char>(encoder));
+
+    request.set("Authorization", "AWS " + access_key_id + ":" + signature.str());
+}
+
+}

dbms/src/IO/S3Common.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <Core/Types.h>
+
+#include <Poco/Net/HTTPRequest.h>
+
+
+namespace DB
+{
+
+struct S3Helper
+{
+    static void authenticateRequest(Poco::Net::HTTPRequest & request,
+        const String & access_key_id,
+        const String & secret_access_key);
+};
+
+}

dbms/src/IO/WriteBufferFromS3.cpp

@@ -1,5 +1,6 @@
 #include <IO/WriteBufferFromS3.h>
 
+#include <IO/S3Common.h>
 #include <IO/WriteHelpers.h>
 
 #include <Poco/DOM/AutoPtr.h>
@@ -113,6 +114,8 @@ void WriteBufferFromS3::initiate()
         request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_POST, initiate_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
         request_ptr->setHost(initiate_uri.getHost()); // use original, not resolved host name in header
 
+        S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
+
         request_ptr->setContentLength(0);
 
         LOG_TRACE((&Logger::get("WriteBufferFromS3")), "Sending request to " << initiate_uri.toString());
@@ -173,6 +176,8 @@ void WriteBufferFromS3::writePart(const String & data)
         request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_PUT, part_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
         request_ptr->setHost(part_uri.getHost()); // use original, not resolved host name in header
 
+        S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
+
         request_ptr->setExpectContinue(true);
 
         request_ptr->setContentLength(data.size());
@@ -240,6 +245,8 @@ void WriteBufferFromS3::complete()
         request_ptr = std::make_unique<Poco::Net::HTTPRequest>(Poco::Net::HTTPRequest::HTTP_POST, complete_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1);
         request_ptr->setHost(complete_uri.getHost()); // use original, not resolved host name in header
 
+        S3Helper::authenticateRequest(*request_ptr, access_key_id, secret_access_key);
+
         request_ptr->setExpectContinue(true);
 
         request_ptr->setContentLength(data.size());