diff --git a/contrib/aws b/contrib/aws
index 06aa8759d17..00b03604543 160000
--- a/contrib/aws
+++ b/contrib/aws
@@ -1 +1 @@
-Subproject commit 06aa8759d17f2032ffd5efa83969270ca9ac727b
+Subproject commit 00b03604543367d7e310cb0993973fdcb723ea79
diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml
index f5e350e5b6b..32d78468a71 100644
--- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml
+++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml
@@ -12,6 +12,15 @@
0
+
+ s3
+
+ http://resolver:8080/root/data/
+ minio
+ minio123
+
+ true
+
@@ -32,6 +41,13 @@
+
+
+
+ s3_retryable
+
+
+
diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py
index 1754b1f175c..3f219b6ba57 100644
--- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py
+++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py
@@ -18,6 +18,16 @@ def fail_request(_request_number):
return 'OK'
+@route('/throttle_request/<_request_number>')
+def fail_request(_request_number):
+ request_number = int(_request_number)
+ if request_number > 0:
+ cache['throttle_request_number'] = request_number
+ else:
+ cache.pop('throttle_request_number', None)
+ return 'OK'
+
+
# Handle for MultipleObjectsDelete.
@route('/<_bucket>', ['POST'])
def delete(_bucket):
@@ -37,6 +47,15 @@ def server(_bucket, _path):
response.content_type = 'text/xml'
return 'ExpectedError
Expected Errortxfbd566d03042474888193-00608d7537'
+ if cache.get('throttle_request_number', None):
+ request_number = cache.pop('throttle_request_number') - 1
+ if request_number > 0:
+ cache['throttle_request_number'] = request_number
+ else:
+ response.status = 429
+ response.content_type = 'text/xml'
+ return 'TooManyRequestsException
Please reduce your request rate.txfbd566d03042474888193-00608d7538'
+
response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path)
response.status = 307
return 'Redirected'
diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py
index 56d9441aba6..b6b47417523 100644
--- a/tests/integration/test_merge_tree_s3_failover/test.py
+++ b/tests/integration/test_merge_tree_s3_failover/test.py
@@ -38,6 +38,12 @@ def fail_request(cluster, request):
assert response == 'OK', 'Expected "OK", but got "{}"'.format(response)
+def throttle_request(cluster, request):
+ response = cluster.exec_in_container(cluster.get_container_id('resolver'),
+ ["curl", "-s", "http://resolver:8080/throttle_request/{}".format(request)])
+ assert response == 'OK', 'Expected "OK", but got "{}"'.format(response)
+
+
@pytest.fixture(scope="module")
def cluster():
try:
@@ -186,3 +192,27 @@ def test_move_failover(cluster):
# Ensure data is not corrupted.
assert node.query("CHECK TABLE s3_failover_test") == '1\n'
assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')"
+
+
+# Check that throttled request retries and does not cause an error on disk with default `retry_attempts` (>0)
+def test_throttle_retry(cluster):
+ node = cluster.instances["node"]
+
+ node.query(
+ """
+ CREATE TABLE s3_throttle_retry_test (
+ id Int64
+ ) ENGINE=MergeTree()
+ ORDER BY id
+ SETTINGS storage_policy='s3_retryable'
+ """
+ )
+
+ data = "(42)"
+ node.query("INSERT INTO s3_throttle_retry_test VALUES {}".format(data))
+
+ throttle_request(cluster, 1)
+
+ assert node.query("""
+ SELECT * FROM s3_throttle_retry_test
+ """) == '42\n'