From bb6bc61f2235581f254e2bc25c20986192e723d7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 21 Apr 2021 19:53:38 +0000 Subject: [PATCH] Fix checks --- src/Storages/StorageExternalDistributed.h | 2 +- src/Storages/StorageURL.cpp | 8 ++-- .../test_redirect_url_storage/test.py | 39 +------------------ 3 files changed, 6 insertions(+), 43 deletions(-) diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index 0724f4bbb45..a6718398a3a 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -53,7 +53,7 @@ protected: ContextPtr context_); StorageExternalDistributed( - const String & uri, + const String & addresses_description, const StorageID & table_id, const String & format_name, const std::optional & format_settings, diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 7e5db5d1b52..dd4b9c4940b 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -271,7 +271,7 @@ Pipe StorageURLWithFailover::read( ConnectionTimeouts::getHTTPTimeouts(local_context), chooseCompressionMethod(request_uri.getPath(), compression_method)); - std::shuffle(uri_options.begin(), uri_options.begin(), thread_local_rng); + std::shuffle(uri_options.begin(), uri_options.end(), thread_local_rng); return Pipe(url_source); } @@ -321,9 +321,9 @@ StorageURLWithFailover::StorageURLWithFailover( { for (const auto & uri_option : uri_options_) { - Poco::URI uri(uri_option); - context_->getRemoteHostFilter().checkURL(uri); - uri_options.emplace_back(std::move(uri)); + Poco::URI uri_(uri_option); + context_->getRemoteHostFilter().checkURL(uri_); + uri_options.emplace_back(std::move(uri_)); LOG_DEBUG(&Poco::Logger::get("StorageURLDistributed"), "Adding URL option: {}", uri_option); } } diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index 549adcd80c1..d899d316abc 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -26,44 +26,7 @@ def test_url_without_redirect(started_cluster): assert node1.query("select * from WebHDFSStorage") == "1\tMark\t72.53\n" -def test_url_with_globs_1(started_cluster): - query = "SELECT DISTINCT(users) FROM (" - for i in range (10, 20): - query += """ - SELECT DISTINCT(users) FROM - ( - SELECT JSONExtractString(data, 'actor', 'login') AS users, - JSONExtractString(data, 'payload', 'comment', 'body') AS body - FROM url('https://data.gharchive.org/2020-11-11-{}.json.gz', TSV, 'data String') - ) - UNION ALL - """.format(i) - - query += """ - SELECT DISTINCT(users) FROM - ( - SELECT JSONExtractString(data, 'actor', 'login') AS users, - JSONExtractString(data, 'payload', 'comment', 'body') AS body - FROM url('https://data.gharchive.org/2020-11-11-20.json.gz', TSV, 'data String') - )) ORDER BY users""" - - expected = node1.query(query) - - result = node1.query(""" - SELECT DISTINCT(users) FROM - ( - SELECT JSONExtractString(data, 'actor', 'login') AS users, - JSONExtractString(data, 'payload', 'comment', 'body') AS body - FROM url('https://data.gharchive.org/2020-11-11-{10..20}.json.gz', TSV, 'data String') - ) - ORDER BY users; - """) - - assert(result == expected) - - -def test_url_with_globs_2(started_cluster): - # 6 addresses +def test_url_with_globs(started_cluster): started_cluster.hdfs_api.write_data("/simple_storage_1_1", "1\n") started_cluster.hdfs_api.write_data("/simple_storage_1_2", "2\n") started_cluster.hdfs_api.write_data("/simple_storage_1_3", "3\n")