mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-05 14:02:21 +00:00
2bade7db08
* initial impl * fix env ut * move ut directory * make sure no null proxy resolver is returned by ProxyConfigurationResolverProvider * minor adjustment * add a few tests, still incomplete * add proxy support for url table function * use proxy for select from url as well * remove optional from return type, just returns empty config * fix style * style * black * ohg boy * rm in progress file * god pls don't let me kill anyone * ... * add use_aws guards * remove hard coded s3 proxy resolver * add concurrency-mt-unsafe * aa * black * add logging back * revert change * imrpove code a bit * helper functions and separate tests * for some reason, this env test is not working.. * formatting * :) * clangtidy * lint * revert some stupid things * small test adjusmtments * simplify tests * rename test * remove extra line * freaking style change * simplify a bit * fix segfault & remove an extra call * tightly couple proxy provider with context.. * remove useless include * rename config prefix parameter * simplify provider a bit * organize provider a bit * add a few comments * comment out proxy env tests * fix nullptr in unit tests * make sure old storage proxy config is properly covered without global context instance * move a few functions from class to anonymous namespace * fix no fallback for specific storage conf * change API to accept http method instead of bool * implement http/https distinction in listresolver, any still not implemented * implement http/https distinction in remote resolver * progress on code, improve tests and add url function working test * use protcol instead of method for http and https * small fix * few more adjustments * fix style * black * move enum to proxyconfiguration * wip * fix build * fix ut * delete atomicroundrobin class * remove stale include * add some tests.. need to spend some more time on the design.. * change design a bit * progress * use existing context for tests * rename aux function and fix ut * .. * rename test * try to simplify tests a bit * simplify tests a bit more * attempt to fix tests, accept more than one remote resolver * use proper log id * try waiting for resolver * proper wait logic * black * empty * address a few comments * refactor tests * remove old tests * baclk * use RAII to set/unset env * black * clang tidy * fix env proxy not respecting any * use log trace * fix wrong logic in getRemoteREsolver * fix wrong logic in getRemoteREsolver * fix test * remove unwanted code * remove ClientConfigurationperRequest and auxilary classes * remove unwanted code * remove adapter test * few adjustments and add test for s3 storage conf with new proxy settings * black * use chassert for context * Add getenv comment
85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
import logging
|
|
import time
|
|
from datetime import datetime
|
|
import hmac
|
|
import hashlib
|
|
import base64
|
|
|
|
import pytest
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def cluster():
|
|
try:
|
|
cluster = ClickHouseCluster(__file__)
|
|
|
|
cluster.add_instance(
|
|
"proxy_list_node",
|
|
main_configs=["configs/config.d/proxy_list.xml"],
|
|
with_minio=True,
|
|
)
|
|
|
|
logging.info("Starting cluster...")
|
|
cluster.start()
|
|
logging.info("Cluster started")
|
|
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def check_proxy_logs(cluster, proxy_instance, http_methods):
|
|
minio_ip = cluster.get_instance_ip("minio1")
|
|
for i in range(10):
|
|
logs = cluster.get_container_logs(proxy_instance)
|
|
# Check with retry that all possible interactions with Minio are present
|
|
for http_method in http_methods:
|
|
method_with_domain = http_method + " http://minio1"
|
|
method_with_ip = http_method + f" http://{minio_ip}"
|
|
|
|
logging.info(f"Method with ip: {method_with_ip}")
|
|
|
|
has_get_minio_logs = (
|
|
logs.find(method_with_domain) >= 0 or logs.find(method_with_ip) >= 0
|
|
)
|
|
if has_get_minio_logs:
|
|
return
|
|
time.sleep(1)
|
|
else:
|
|
assert False, "http method not found in logs"
|
|
|
|
|
|
def test_s3_with_proxy_list(cluster):
|
|
node = cluster.instances["proxy_list_node"]
|
|
|
|
# insert into function url uses POST and minio expects PUT
|
|
node.query(
|
|
"""
|
|
INSERT INTO FUNCTION
|
|
s3('http://minio1:9001/root/data/ch-proxy-test/test.csv', 'minio', 'minio123', 'CSV', 'key String, value String')
|
|
VALUES ('color','red'),('size','10')
|
|
"""
|
|
)
|
|
|
|
content_type = "application/zstd"
|
|
date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
|
|
resource = "/root/data/ch-proxy-test/test.csv"
|
|
get_sig_string = f"GET\n\n{content_type}\n{date}\n{resource}"
|
|
password = "minio123"
|
|
|
|
get_digest = hmac.new(
|
|
password.encode("utf-8"), get_sig_string.encode("utf-8"), hashlib.sha1
|
|
).digest()
|
|
get_signature = base64.b64encode(get_digest).decode("utf-8")
|
|
assert (
|
|
node.query(
|
|
"SELECT * FROM url('http://minio1:9001/root/data/ch-proxy-test/test.csv', 'CSV', 'a String, b String',"
|
|
f"headers('Host'='minio1', 'Date'= '{date}', 'Content-Type'='{content_type}',"
|
|
f"'Authorization'='AWS minio:{get_signature}')) FORMAT Values"
|
|
)
|
|
== "('color','red'),('size','10')"
|
|
)
|
|
|
|
check_proxy_logs(cluster, "proxy1", ["GET"])
|