Added region parameter for S3 storage and disk.

src/Disks/S3/registerDiskS3.cpp

@@ -115,6 +115,7 @@ std::shared_ptr<Aws::S3::S3Client>
 getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextConstPtr context)
 {
     S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
+        config.getString(config_prefix + ".region"),
         context->getRemoteHostFilter(), context->getGlobalContext()->getSettingsRef().s3_max_redirects);
 
     S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint")));

src/IO/S3/PocoHTTPClient.cpp

@@ -47,9 +47,11 @@ namespace DB::S3
 {
 
 PocoHTTPClientConfiguration::PocoHTTPClientConfiguration(
+        const String & force_region_,
         const RemoteHostFilter & remote_host_filter_,
         unsigned int s3_max_redirects_)
-    : remote_host_filter(remote_host_filter_)
+    : force_region(force_region_)
+    , remote_host_filter(remote_host_filter_)
     , s3_max_redirects(s3_max_redirects_)
 {
 }
@@ -63,15 +65,23 @@ void PocoHTTPClientConfiguration::updateSchemeAndRegion()
         if (uri.getScheme() == "http")
             scheme = Aws::Http::Scheme::HTTP;
 
-        String matched_region;
+        if (force_region.empty())
-        if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region))
         {
-            boost::algorithm::to_lower(matched_region);
+            String matched_region;
-            region = matched_region;
+            if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region))
+            {
+                boost::algorithm::to_lower(matched_region);
+                region = matched_region;
+            }
+            else
+            {
+                /// In global mode AWS C++ SDK send `us-east-1` but accept switching to another one if being suggested.
+                region = Aws::Region::AWS_GLOBAL;
+            }
         }
         else
         {
-            region = Aws::Region::AWS_GLOBAL;
+            region = force_region;
         }
     }
 }

src/IO/S3/PocoHTTPClient.h

@@ -29,13 +29,14 @@ class ClientFactory;
 
 struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration
 {
+    String force_region;
     const RemoteHostFilter & remote_host_filter;
     unsigned int s3_max_redirects;
 
     void updateSchemeAndRegion();
 
 private:
-    PocoHTTPClientConfiguration(const RemoteHostFilter & remote_host_filter_, unsigned int s3_max_redirects_);
+    PocoHTTPClientConfiguration(const String & force_region_, const RemoteHostFilter & remote_host_filter_, unsigned int s3_max_redirects_);
 
     /// Constructor of Aws::Client::ClientConfiguration must be called after AWS SDK initialization.
     friend ClientFactory;

src/IO/S3Common.cpp

@@ -410,7 +410,7 @@ public:
             }
             else if (Aws::Utils::StringUtils::ToLower(ec2_metadata_disabled.c_str()) != "true")
             {
-                DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(configuration.remote_host_filter, configuration.s3_max_redirects);
+                DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(configuration.region, configuration.remote_host_filter, configuration.s3_max_redirects);
 
                 /// See MakeDefaultHttpResourceClientConfiguration().
                 /// This is part of EC2 metadata client, but unfortunately it can't be accessed from outside
@@ -590,10 +590,11 @@ namespace S3
     }
 
     PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT
+        const String & force_region,
         const RemoteHostFilter & remote_host_filter,
         unsigned int s3_max_redirects)
     {
-        return PocoHTTPClientConfiguration(remote_host_filter, s3_max_redirects);
+        return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects);
     }
 
     URI::URI(const Poco::URI & uri_)

src/IO/S3Common.h

@@ -42,6 +42,7 @@ public:
         bool use_insecure_imds_request);
 
     PocoHTTPClientConfiguration createClientConfiguration(
+        const String & force_region,
         const RemoteHostFilter & remote_host_filter,
         unsigned int s3_max_redirects);
 

src/Storages/StorageS3.cpp

@@ -444,6 +444,7 @@ void StorageS3::updateClientAndAuthSettings(ContextPtr ctx, StorageS3::ClientAut
     }
 
     S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration(
+        settings.region,
         ctx->getRemoteHostFilter(), ctx->getGlobalContext()->getSettingsRef().s3_max_redirects);
 
     client_configuration.endpointOverride = upd.uri.endpoint;

src/Storages/StorageS3Settings.cpp

@@ -30,6 +30,7 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
             auto endpoint = config.getString(config_elem + "." + key + ".endpoint");
             auto access_key_id = config.getString(config_elem + "." + key + ".access_key_id", "");
             auto secret_access_key = config.getString(config_elem + "." + key + ".secret_access_key", "");
+            auto region = config.getString(config_elem + "." + key + ".region", "");
             auto server_side_encryption_customer_key_base64 = config.getString(config_elem + "." + key + ".server_side_encryption_customer_key_base64", "");
             std::optional<bool> use_environment_credentials;
             if (config.has(config_elem + "." + key + ".use_environment_credentials"))
@@ -57,7 +58,14 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
                 }
             }
 
-            settings.emplace(endpoint, S3AuthSettings{std::move(access_key_id), std::move(secret_access_key), std::move(server_side_encryption_customer_key_base64), std::move(headers), use_environment_credentials, use_insecure_imds_request});
+            settings.emplace(endpoint, S3AuthSettings{
+                    std::move(access_key_id), std::move(secret_access_key),
+                    std::move(region),
+                    std::move(server_side_encryption_customer_key_base64),
+                    std::move(headers),
+                    use_environment_credentials,
+                    use_insecure_imds_request
+                });
         }
     }
 }

src/Storages/StorageS3Settings.h

@@ -28,6 +28,7 @@ struct S3AuthSettings
 {
     String access_key_id;
     String secret_access_key;
+    String region;
     String server_side_encryption_customer_key_base64;
 
     HeaderCollection headers;
@@ -38,7 +39,9 @@ struct S3AuthSettings
     inline bool operator==(const S3AuthSettings & other) const
     {
         return access_key_id == other.access_key_id && secret_access_key == other.secret_access_key
-            && server_side_encryption_customer_key_base64 == other.server_side_encryption_customer_key_base64 && headers == other.headers
+            && region == other.region
+            && server_side_encryption_customer_key_base64 == other.server_side_encryption_customer_key_base64
+            && headers == other.headers
             && use_environment_credentials == other.use_environment_credentials
             && use_insecure_imds_request == other.use_insecure_imds_request;
     }