diff --git a/contrib/aws b/contrib/aws index 06aa8759d17..00b03604543 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 06aa8759d17f2032ffd5efa83969270ca9ac727b +Subproject commit 00b03604543367d7e310cb0993973fdcb723ea79 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index f5e350e5b6b..32d78468a71 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -12,6 +12,15 @@ 0 + + s3 + + http://resolver:8080/root/data/ + minio + minio123 + + true + @@ -32,6 +41,13 @@ + + +
+ s3_retryable +
+
+
diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 1754b1f175c..3f219b6ba57 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -18,6 +18,16 @@ def fail_request(_request_number): return 'OK' +@route('/throttle_request/<_request_number>') +def fail_request(_request_number): + request_number = int(_request_number) + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + cache.pop('throttle_request_number', None) + return 'OK' + + # Handle for MultipleObjectsDelete. @route('/<_bucket>', ['POST']) def delete(_bucket): @@ -37,6 +47,15 @@ def server(_bucket, _path): response.content_type = 'text/xml' return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' + if cache.get('throttle_request_number', None): + request_number = cache.pop('throttle_request_number') - 1 + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + response.status = 429 + response.content_type = 'text/xml' + return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) response.status = 307 return 'Redirected' diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 56d9441aba6..b6b47417523 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -38,6 +38,12 @@ def fail_request(cluster, request): assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) +def throttle_request(cluster, request): + response = cluster.exec_in_container(cluster.get_container_id('resolver'), + ["curl", "-s", "http://resolver:8080/throttle_request/{}".format(request)]) + assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) + + @pytest.fixture(scope="module") def cluster(): try: @@ -186,3 +192,27 @@ def test_move_failover(cluster): # Ensure data is not corrupted. assert node.query("CHECK TABLE s3_failover_test") == '1\n' assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')" + + +# Check that throttled request retries and does not cause an error on disk with default `retry_attempts` (>0) +def test_throttle_retry(cluster): + node = cluster.instances["node"] + + node.query( + """ + CREATE TABLE s3_throttle_retry_test ( + id Int64 + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='s3_retryable' + """ + ) + + data = "(42)" + node.query("INSERT INTO s3_throttle_retry_test VALUES {}".format(data)) + + throttle_request(cluster, 1) + + assert node.query(""" + SELECT * FROM s3_throttle_retry_test + """) == '42\n'