thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 16:50:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23844 from excitoon-favorites/s3morecorrectclient

Browse Source 
Better handling of HTTP errors in `PocoHTTPClient`
This commit is contained in:
alexey-milovidov 2021-05-13 19:21:09 +03:00 committed by GitHub
commit d340e33e2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 15 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/IO/S3/PocoHTTPClient.cpp
View File

@ -289,29 +289,16 @@ void PocoHTTPClient::makeRequestInternal(
} }
LOG_DEBUG(log, "Received headers: {}", headers_ss.str()); LOG_DEBUG(log, "Received headers: {}", headers_ss.str());
if (status_code >= 300) if (status_code == 429 || status_code == 503)
{ { // API throttling
String error_message; ProfileEvents::increment(select_metric(S3MetricType::Throttling));
Poco::StreamCopier::copyToString(response_body_stream, error_message);
if (Aws::Http::IsRetryableHttpResponseCode(response->GetResponseCode()))
response->SetClientErrorType(Aws::Client::CoreErrors::NETWORK_CONNECTION);
else
response->SetClientErrorType(Aws::Client::CoreErrors::USER_CANCELLED);
response->SetClientErrorMessage(error_message);
if (status_code == 429 || status_code == 503)
{ // API throttling
ProfileEvents::increment(select_metric(S3MetricType::Throttling));
}
else
{
ProfileEvents::increment(select_metric(S3MetricType::Errors));
}
} }
else else if (status_code >= 300)
response->SetResponseBody(response_body_stream, session); {
ProfileEvents::increment(select_metric(S3MetricType::Errors));
}
response->SetResponseBody(response_body_stream, session);
return; return;
} }

3
tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py
View File

@ -34,7 +34,8 @@ def server(_bucket, _path):
cache['request_number'] = request_number cache['request_number'] = request_number
else: else:
response.status = 500 response.status = 500
return 'Expected Error' response.content_type = 'text/xml'
return '<?xml version="1.0" encoding="UTF-8"?><Error><Code>ExpectedError</Code><Message>Expected Error</Message><RequestId>txfbd566d03042474888193-00608d7537</RequestId></Error>'
response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path)
response.status = 307 response.status = 307

4
tests/integration/test_storage_s3/s3_mocks/mock_s3.py
View File

@ -15,7 +15,9 @@ def server(_bucket, _path):
for name in request.headers: for name in request.headers:
if name == 'Authorization' and request.headers[name] == 'Bearer TOKEN': if name == 'Authorization' and request.headers[name] == 'Bearer TOKEN':
return '1, 2, 3' return '1, 2, 3'
abort(403) response.status = 403
response.content_type = 'text/xml'
return '<?xml version="1.0" encoding="UTF-8"?><Error><Code>ForbiddenError</Code><Message>Forbidden Error</Message><RequestId>txfbd566d03042474888193-00608d7537</RequestId></Error>'
@route('/') @route('/')

2
tests/integration/test_storage_s3/test.py
View File

@ -504,7 +504,7 @@ def test_custom_auth_headers_exclusion(cluster):
print(result) print(result)
assert ei.value.returncode == 243 assert ei.value.returncode == 243
assert '403 Forbidden' in ei.value.stderr assert 'Forbidden Error' in ei.value.stderr
def test_infinite_redirect(cluster): def test_infinite_redirect(cluster):