diff --git a/base/poco/Net/include/Poco/Net/MessageHeader.h b/base/poco/Net/include/Poco/Net/MessageHeader.h index bbfd849b304..851eae57436 100644 --- a/base/poco/Net/include/Poco/Net/MessageHeader.h +++ b/base/poco/Net/include/Poco/Net/MessageHeader.h @@ -103,6 +103,26 @@ namespace Net /// /// The default limit is 100. + int getNameLengthLimit() const; + /// Returns the maximum length of a field name. + /// + /// See setNameLengthLimit() for more information. + + void setNameLengthLimit(int limit); + /// Sets the maximum length of a field name. + /// + /// The default limit is 256. + + int getValueLengthLimit() const; + /// Returns the maximum length of a field value. + /// + /// See setValueLengthLimit() for more information. + + void setValueLengthLimit(int limit); + /// Sets the maximum length of a field value. + /// + /// The default limit is 8192. + bool hasToken(const std::string & fieldName, const std::string & token) const; /// Returns true iff the field with the given fieldName contains /// the given token. Tokens in a header field are expected to be @@ -157,12 +177,14 @@ namespace Net enum Limits /// Limits for basic sanity checks when reading a header { - MAX_NAME_LENGTH = 256, - MAX_VALUE_LENGTH = 8192, + DFL_NAME_LENGTH_LIMIT = 256, + DFL_VALUE_LENGTH_LIMIT = 8192, DFL_FIELD_LIMIT = 100 }; int _fieldLimit; + int _nameLengthLimit; + int _valueLengthLimit; }; diff --git a/base/poco/Net/src/MessageHeader.cpp b/base/poco/Net/src/MessageHeader.cpp index 0dbfa3e5cbd..1fcd78a9068 100644 --- a/base/poco/Net/src/MessageHeader.cpp +++ b/base/poco/Net/src/MessageHeader.cpp @@ -28,14 +28,18 @@ namespace Net { MessageHeader::MessageHeader(): - _fieldLimit(DFL_FIELD_LIMIT) + _fieldLimit(DFL_FIELD_LIMIT), + _nameLengthLimit(DFL_NAME_LENGTH_LIMIT), + _valueLengthLimit(DFL_VALUE_LENGTH_LIMIT) { } MessageHeader::MessageHeader(const MessageHeader& messageHeader): NameValueCollection(messageHeader), - _fieldLimit(DFL_FIELD_LIMIT) + _fieldLimit(DFL_FIELD_LIMIT), + _nameLengthLimit(DFL_NAME_LENGTH_LIMIT), + _valueLengthLimit(DFL_VALUE_LENGTH_LIMIT) { } @@ -80,12 +84,12 @@ void MessageHeader::read(std::istream& istr) throw MessageException("Too many header fields"); name.clear(); value.clear(); - while (ch != eof && ch != ':' && ch != '\n' && name.length() < MAX_NAME_LENGTH) { name += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != ':' && ch != '\n' && name.length() < _nameLengthLimit) { name += ch; ch = buf.sbumpc(); } if (ch == '\n') { ch = buf.sbumpc(); continue; } // ignore invalid header lines if (ch != ':') throw MessageException("Field name too long/no colon found"); if (ch != eof) ch = buf.sbumpc(); // ':' while (ch != eof && Poco::Ascii::isSpace(ch) && ch != '\r' && ch != '\n') ch = buf.sbumpc(); - while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); } if (ch == '\r') ch = buf.sbumpc(); if (ch == '\n') ch = buf.sbumpc(); @@ -93,7 +97,7 @@ void MessageHeader::read(std::istream& istr) throw MessageException("Field value too long/no CRLF found"); while (ch == ' ' || ch == '\t') // folding { - while (ch != eof && ch != '\r' && ch != '\n' && value.length() < MAX_VALUE_LENGTH) { value += ch; ch = buf.sbumpc(); } + while (ch != eof && ch != '\r' && ch != '\n' && value.length() < _valueLengthLimit) { value += ch; ch = buf.sbumpc(); } if (ch == '\r') ch = buf.sbumpc(); if (ch == '\n') ch = buf.sbumpc(); @@ -122,6 +126,32 @@ void MessageHeader::setFieldLimit(int limit) } +int MessageHeader::getNameLengthLimit() const +{ + return _nameLengthLimit; +} + +void MessageHeader::setNameLengthLimit(int limit) +{ + poco_assert(limit >= 0); + + _nameLengthLimit = limit; +} + + +int MessageHeader::getValueLengthLimit() const +{ + return _valueLengthLimit; +} + +void MessageHeader::setValueLengthLimit(int limit) +{ + poco_assert(limit >= 0); + + _valueLengthLimit = limit; +} + + bool MessageHeader::hasToken(const std::string& fieldName, const std::string& token) const { std::string field = get(fieldName, ""); diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 7dacd8102cc..e0df983c8b9 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -52,6 +52,9 @@ namespace S3RequestSetting { extern const S3RequestSettingsBool allow_native_copy; extern const S3RequestSettingsString storage_class_name; + extern const S3RequestSettingsUInt64 http_max_fields; + extern const S3RequestSettingsUInt64 http_max_field_name_size; + extern const S3RequestSettingsUInt64 http_max_field_value_size; } namespace ErrorCodes @@ -100,6 +103,9 @@ namespace client_configuration.requestTimeoutMs = 60 * 60 * 1000; client_configuration.http_keep_alive_timeout = S3::DEFAULT_KEEP_ALIVE_TIMEOUT; client_configuration.http_keep_alive_max_requests = S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS; + client_configuration.http_max_fields = request_settings[S3RequestSetting::http_max_fields]; + client_configuration.http_max_field_name_size = request_settings[S3RequestSetting::http_max_field_name_size]; + client_configuration.http_max_field_value_size = request_settings[S3RequestSetting::http_max_field_value_size]; S3::ClientSettings client_settings{ .use_virtual_addressing = s3_uri.is_virtual_hosted_style, diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 3e060e21c51..b53058a42da 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -163,6 +163,9 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) + , http_max_fields(client_configuration.http_max_fields) + , http_max_field_name_size(client_configuration.http_max_field_name_size) + , http_max_field_value_size(client_configuration.http_max_field_value_size) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -466,6 +469,9 @@ void PocoHTTPClient::makeRequestInternalImpl( } Poco::Net::HTTPResponse poco_response; + poco_response.setFieldLimit(static_cast(http_max_fields)); + poco_response.setNameLengthLimit(static_cast(http_max_field_name_size)); + poco_response.setValueLengthLimit(static_cast(http_max_field_value_size)); Stopwatch watch; diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index eb65460ce13..f62371706c3 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -57,6 +57,10 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT; size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST; + UInt64 http_max_fields = 1000000; + UInt64 http_max_field_name_size = 128 * 1024; + UInt64 http_max_field_value_size = 128 * 1024; + std::function error_report; void updateSchemeAndRegion(); @@ -177,6 +181,9 @@ protected: const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects = 0; bool s3_use_adaptive_timeouts = true; + const UInt64 http_max_fields = 1000000; + const UInt64 http_max_field_name_size = 128 * 1024; + const UInt64 http_max_field_value_size = 128 * 1024; bool enable_s3_requests_logging = false; bool for_disk_s3 = false; diff --git a/src/IO/S3RequestSettings.cpp b/src/IO/S3RequestSettings.cpp index e35bcfdbc4d..08eb9aaec5a 100644 --- a/src/IO/S3RequestSettings.cpp +++ b/src/IO/S3RequestSettings.cpp @@ -37,7 +37,10 @@ namespace ErrorCodes DECLARE(Bool, check_objects_after_upload, S3::DEFAULT_CHECK_OBJECTS_AFTER_UPLOAD, "", 0) \ DECLARE(Bool, throw_on_zero_files_match, false, "", 0) \ DECLARE(UInt64, max_single_operation_copy_size, S3::DEFAULT_MAX_SINGLE_OPERATION_COPY_SIZE, "", 0) \ - DECLARE(String, storage_class_name, "", "", 0) + DECLARE(String, storage_class_name, "", "", 0) \ + DECLARE(UInt64, http_max_fields, 1000000, "", 0) \ + DECLARE(UInt64, http_max_field_name_size, 128 * 1024, "", 0) \ + DECLARE(UInt64, http_max_field_value_size, 128 * 1024, "", 0) #define PART_UPLOAD_SETTINGS(DECLARE, ALIAS) \ DECLARE(UInt64, strict_upload_part_size, 0, "", 0) \