From e5e84960eaa0a61e220f9a19a815edb970b2ccbd Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 29 Apr 2020 11:39:00 +0300 Subject: [PATCH] S3 with proxy integration test and fixes. --- .../compose/docker_compose_minio.yml | 11 ++++ src/Disks/S3/DynamicProxyConfiguration.cpp | 5 +- src/Disks/S3/registerDiskS3.cpp | 19 ++++-- tests/integration/helpers/cluster.py | 9 +++ .../test_s3_with_proxy/__init__.py | 0 .../configs/config.d/log_conf.xml | 12 ++++ .../configs/config.d/storage_conf.xml | 25 ++++++++ .../configs/config.d/users.xml | 5 ++ .../test_s3_with_proxy/configs/config.xml | 20 ++++++ tests/integration/test_s3_with_proxy/test.py | 64 +++++++++++++++++++ 10 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_s3_with_proxy/__init__.py create mode 100644 tests/integration/test_s3_with_proxy/configs/config.d/log_conf.xml create mode 100644 tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml create mode 100644 tests/integration/test_s3_with_proxy/configs/config.d/users.xml create mode 100644 tests/integration/test_s3_with_proxy/configs/config.xml create mode 100644 tests/integration/test_s3_with_proxy/test.py diff --git a/docker/test/integration/compose/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml index 41816427794..5b93369e3e2 100644 --- a/docker/test/integration/compose/docker_compose_minio.yml +++ b/docker/test/integration/compose/docker_compose_minio.yml @@ -27,5 +27,16 @@ services: - SERVER_REDIRECT_CODE=307 - SERVER_ACCESS_LOG=/nginx/access.log +# HTTP proxies for Minio. + proxy1: + image: vimagick/tinyproxy + ports: + - "4081:8888" + + proxy2: + image: vimagick/tinyproxy + ports: + - "4082:8888" + volumes: data1-1: diff --git a/src/Disks/S3/DynamicProxyConfiguration.cpp b/src/Disks/S3/DynamicProxyConfiguration.cpp index 62930bf5fa7..71765faa021 100644 --- a/src/Disks/S3/DynamicProxyConfiguration.cpp +++ b/src/Disks/S3/DynamicProxyConfiguration.cpp @@ -1,6 +1,7 @@ #include "DynamicProxyConfiguration.h" #include +#include namespace DB::S3 { @@ -18,7 +19,9 @@ Aws::Client::ClientConfigurationPerRequest DynamicProxyConfiguration::getConfigu cfg.proxyHost = proxies[index].getHost(); cfg.proxyPort = proxies[index].getPort(); - return cfg; + LOG_DEBUG(&Logger::get("AWSClient"), "Use proxy: " << proxies[index].toString()); + + return cfg; } } diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 75ba89515cf..d45c7c1efa3 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -37,18 +37,29 @@ namespace std::unique_ptr getProxyConfiguration(const Poco::Util::AbstractConfiguration * config) { - if (config->has("proxies")) + if (config->has("proxy")) { std::vector keys; - config->keys("proxies", keys); + config->keys("proxy", keys); std::vector proxies; for (const auto & key : keys) if (startsWith(key, "uri")) - proxies.emplace_back(config->getString("proxies." + key)); + { + Poco::URI proxy_uri(config->getString("proxy." + key)); + + if (proxy_uri.getScheme() != "http") + throw Exception("Only HTTP scheme is allowed in proxy configuration at the moment, proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); + if (proxy_uri.getHost().empty()) + throw Exception("Empty host in proxy configuration, proxy uri: " + proxy_uri.toString(), ErrorCodes::BAD_ARGUMENTS); + + proxies.push_back(proxy_uri); + + LOG_DEBUG(&Logger::get("DiskS3"), "Configured proxy: " << proxy_uri.toString()); + } if (!proxies.empty()) - return std::make_unique(proxies); + return std::make_unique(proxies); } return nullptr; } diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 8a5fed76377..17f3ed76ed5 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -300,6 +300,15 @@ class ClickHouseCluster: handle = self.docker_client.containers.get(docker_id) return handle.attrs['NetworkSettings']['Networks'].values()[0]['IPAddress'] + def get_container_id(self, instance_name): + docker_id = self.get_instance_docker_id(instance_name) + handle = self.docker_client.containers.get(docker_id) + return handle.attrs['Id'] + + def get_container_logs(self, instance_name): + container_id = self.get_container_id(instance_name) + return self.docker_client.api.logs(container_id) + def wait_mysql_to_start(self, timeout=60): start = time.time() while time.time() - start < timeout: diff --git a/tests/integration/test_s3_with_proxy/__init__.py b/tests/integration/test_s3_with_proxy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/log_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/tests/integration/test_s3_with_proxy/configs/config.d/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..7827eec4498 --- /dev/null +++ b/tests/integration/test_s3_with_proxy/configs/config.d/storage_conf.xml @@ -0,0 +1,25 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + http://proxy1:8888 + http://proxy2:8888 + + + + + + +
+ s3 +
+ + + + + diff --git a/tests/integration/test_s3_with_proxy/configs/config.d/users.xml b/tests/integration/test_s3_with_proxy/configs/config.d/users.xml new file mode 100644 index 00000000000..797113053f4 --- /dev/null +++ b/tests/integration/test_s3_with_proxy/configs/config.d/users.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/integration/test_s3_with_proxy/configs/config.xml b/tests/integration/test_s3_with_proxy/configs/config.xml new file mode 100644 index 00000000000..24b7344df3a --- /dev/null +++ b/tests/integration/test_s3_with_proxy/configs/config.xml @@ -0,0 +1,20 @@ + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py new file mode 100644 index 00000000000..8881d0e4b74 --- /dev/null +++ b/tests/integration/test_s3_with_proxy/test.py @@ -0,0 +1,64 @@ +import logging + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +# Creates S3 bucket for tests and allows anonymous read-write access to it. +def prepare_s3_bucket(cluster): + minio_client = cluster.minio_client + + if minio_client.bucket_exists(cluster.minio_bucket): + minio_client.remove_bucket(cluster.minio_bucket) + + minio_client.make_bucket(cluster.minio_bucket) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance("node", config_dir="configs", with_minio=True) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + prepare_s3_bucket(cluster) + logging.info("S3 bucket created") + + yield cluster + finally: + cluster.shutdown() + + +def check_proxy_logs(cluster, proxy_instance): + logs = cluster.get_container_logs(proxy_instance) + # Check that all possible interactions with Minio are present + for http_method in ["POST", "PUT", "GET", "DELETE"]: + assert logs.find(http_method + " http://minio1") >= 0 + + +def test_s3_with_proxy_list(cluster): + node = cluster.instances["node"] + + node.query( + """ + CREATE TABLE s3_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='s3' + """ + ) + + node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") + assert node.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + + node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + + for proxy in ["proxy1", "proxy2"]: + check_proxy_logs(cluster, proxy)