Backport #72778 to 24.10: Add S3 request settings http_max_fields, http_max_field_name_size, http_max_field_value_size for parsing S3 API responses

base/poco/Net/include/Poco/Net/MessageHeader.h

@@ -103,6 +103,26 @@ namespace Net
         ///
         /// The default limit is 100.
 
+        int getNameLengthLimit() const;
+		/// Returns the maximum length of a field name.
+		///
+		/// See setNameLengthLimit() for more information.
+
+        void setNameLengthLimit(int limit);
+		/// Sets the maximum length of a field name.
+		///
+		/// The default limit is 256.
+
+        int getValueLengthLimit() const;
+		/// Returns the maximum length of a field value.
+		///
+		/// See setValueLengthLimit() for more information.
+
+        void setValueLengthLimit(int limit);
+		/// Sets the maximum length of a field value.
+		///
+		/// The default limit is 8192.
+
         bool hasToken(const std::string & fieldName, const std::string & token) const;
         /// Returns true iff the field with the given fieldName contains
         /// the given token. Tokens in a header field are expected to be
@@ -157,12 +177,14 @@ namespace Net
         enum Limits
         /// Limits for basic sanity checks when reading a header
         {
-            MAX_NAME_LENGTH = 256,
+            DFL_NAME_LENGTH_LIMIT = 256,
-            MAX_VALUE_LENGTH = 8192,
+            DFL_VALUE_LENGTH_LIMIT = 8192,
             DFL_FIELD_LIMIT = 100
         };
 
         int _fieldLimit;
+        int _nameLengthLimit;
+        int _valueLengthLimit;
     };
 
 

base/poco/Net/src/MessageHeader.cpp

@@ -28,14 +28,18 @@ namespace Net {
 
 
 MessageHeader::MessageHeader():
-	_fieldLimit(DFL_FIELD_LIMIT)
+	_fieldLimit(DFL_FIELD_LIMIT),
+	_nameLengthLimit(DFL_NAME_LENGTH_LIMIT),
+	_valueLengthLimit(DFL_VALUE_LENGTH_LIMIT)
 {
 }
 
 
 MessageHeader::MessageHeader(const MessageHeader& messageHeader):
 	NameValueCollection(messageHeader),
-	_fieldLimit(DFL_FIELD_LIMIT)
+	_fieldLimit(DFL_FIELD_LIMIT),
+	_nameLengthLimit(DFL_NAME_LENGTH_LIMIT),
+	_valueLengthLimit(DFL_VALUE_LENGTH_LIMIT)
 {
 }
 
@@ -80,12 +84,12 @@ void MessageHeader::read(std::istream& istr)
 			throw MessageException("Too many header fields");
 		name.clear();
 		value.clear();
-		while (ch != eof && ch != ':' && ch != '\n' && name.length() < MAX_NAME_LENGTH) { name += ch; ch = buf.sbumpc(); }
+		while (ch != eof && ch != ':' && ch != '\n' && name.length() < _nameLengthLimit) { name += ch; ch = buf.sbumpc(); }
 		if (ch == '\n') { ch = buf.sbumpc(); continue; } // ignore invalid header lines
 		if (ch != ':') throw MessageException("Field name too long/no colon found");
 		if (ch != eof) ch = buf.sbumpc(); // ':'
 		while (ch != eof && Poco::Ascii::isSpace(ch) && ch != '\r' && ch != '\n') ch = buf.sbumpc();
-		while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); }
+		while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); }
 		if (ch == '\r') ch = buf.sbumpc();
 		if (ch == '\n')
 			ch = buf.sbumpc();
@@ -93,7 +97,7 @@ void MessageHeader::read(std::istream& istr)
 			throw MessageException("Field value too long/no CRLF found");
 		while (ch == ' ' || ch == '\t') // folding
 		{
-			while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); }
+			while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); }
 			if (ch == '\r') ch = buf.sbumpc();
 			if (ch == '\n')
 				ch = buf.sbumpc();
@@ -122,6 +126,32 @@ void MessageHeader::setFieldLimit(int limit)
 }
 
 
+int MessageHeader::getNameLengthLimit() const
+{
+	return _nameLengthLimit;
+}
+
+void MessageHeader::setNameLengthLimit(int limit)
+{
+	poco_assert(limit >= 0);
+
+	_nameLengthLimit = limit;
+}
+
+
+int MessageHeader::getValueLengthLimit() const
+{
+	return _valueLengthLimit;
+}
+
+void MessageHeader::setValueLengthLimit(int limit)
+{
+	poco_assert(limit >= 0);
+
+	_valueLengthLimit = limit;
+}
+
+
 bool MessageHeader::hasToken(const std::string& fieldName, const std::string& token) const
 {
 	std::string field = get(fieldName, "");

src/Backups/BackupIO_S3.cpp

@@ -52,6 +52,9 @@ namespace S3RequestSetting
 {
     extern const S3RequestSettingsBool allow_native_copy;
     extern const S3RequestSettingsString storage_class_name;
+    extern const S3RequestSettingsUInt64 http_max_fields;
+    extern const S3RequestSettingsUInt64 http_max_field_name_size;
+    extern const S3RequestSettingsUInt64 http_max_field_value_size;
 }
 
 namespace ErrorCodes
@@ -100,6 +103,9 @@ namespace
         client_configuration.requestTimeoutMs = 60 * 60 * 1000;
         client_configuration.http_keep_alive_timeout = S3::DEFAULT_KEEP_ALIVE_TIMEOUT;
         client_configuration.http_keep_alive_max_requests = S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS;
+        client_configuration.http_max_fields = request_settings[S3RequestSetting::http_max_fields];
+        client_configuration.http_max_field_name_size = request_settings[S3RequestSetting::http_max_field_name_size];
+        client_configuration.http_max_field_value_size = request_settings[S3RequestSetting::http_max_field_value_size];
 
         S3::ClientSettings client_settings{
             .use_virtual_addressing = s3_uri.is_virtual_hosted_style,

src/IO/S3/PocoHTTPClient.cpp

@@ -163,6 +163,9 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config
     , remote_host_filter(client_configuration.remote_host_filter)
     , s3_max_redirects(client_configuration.s3_max_redirects)
     , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts)
+    , http_max_fields(client_configuration.http_max_fields)
+    , http_max_field_name_size(client_configuration.http_max_field_name_size)
+    , http_max_field_value_size(client_configuration.http_max_field_value_size)
     , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging)
     , for_disk_s3(client_configuration.for_disk_s3)
     , get_request_throttler(client_configuration.get_request_throttler)
@@ -466,6 +469,9 @@ void PocoHTTPClient::makeRequestInternalImpl(
             }
 
             Poco::Net::HTTPResponse poco_response;
+            poco_response.setFieldLimit(static_cast<int>(http_max_fields));
+            poco_response.setNameLengthLimit(static_cast<int>(http_max_field_name_size));
+            poco_response.setValueLengthLimit(static_cast<int>(http_max_field_value_size));
 
             Stopwatch watch;
 

src/IO/S3/PocoHTTPClient.h

@@ -57,6 +57,10 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration
     size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT;
     size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST;
 
+    UInt64 http_max_fields = 1000000;
+    UInt64 http_max_field_name_size = 128 * 1024;
+    UInt64 http_max_field_value_size = 128 * 1024;
+
     std::function<void(const ProxyConfiguration &)> error_report;
 
     void updateSchemeAndRegion();
@@ -177,6 +181,9 @@ protected:
     const RemoteHostFilter & remote_host_filter;
     unsigned int s3_max_redirects = 0;
     bool s3_use_adaptive_timeouts = true;
+    const UInt64 http_max_fields = 1000000;
+    const UInt64 http_max_field_name_size = 128 * 1024;
+    const UInt64 http_max_field_value_size = 128 * 1024;
     bool enable_s3_requests_logging = false;
     bool for_disk_s3 = false;
 

src/IO/S3RequestSettings.cpp

@@ -37,7 +37,10 @@ namespace ErrorCodes
     DECLARE(Bool, check_objects_after_upload, S3::DEFAULT_CHECK_OBJECTS_AFTER_UPLOAD, "", 0) \
     DECLARE(Bool, throw_on_zero_files_match, false, "", 0) \
     DECLARE(UInt64, max_single_operation_copy_size, S3::DEFAULT_MAX_SINGLE_OPERATION_COPY_SIZE, "", 0) \
-    DECLARE(String, storage_class_name, "", "", 0)
+    DECLARE(String, storage_class_name, "", "", 0) \
+    DECLARE(UInt64, http_max_fields, 1000000, "", 0) \
+    DECLARE(UInt64, http_max_field_name_size, 128 * 1024, "", 0) \
+    DECLARE(UInt64, http_max_field_value_size, 128 * 1024, "", 0)
 
 #define PART_UPLOAD_SETTINGS(DECLARE, ALIAS) \
     DECLARE(UInt64, strict_upload_part_size, 0, "", 0) \