From 52f242daf056a4d6725de1b87fd7436769ede79e Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Fri, 31 May 2019 18:16:40 +0000 Subject: [PATCH] tests (WIP) --- .../clickhouse-test | 126 ++++++++++++++++++ .../00950_table_function_s3_wip/config.xml | 115 ++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100755 dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test create mode 100644 dbms/tests/queries/0_stateless/00950_table_function_s3_wip/config.xml diff --git a/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test b/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test new file mode 100755 index 00000000000..09876ea0df6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +import http.server +import os +import subprocess +import threading +import unittest + + +format = 'column1 UInt32, column2 UInt32, column3 UInt32' +values = '(1, 2, 3), (2, 3, 1), (78, 43, 45)' +redirecting_host = '127.0.0.1' +redirecting_to_http_port = 12345 +redirecting_to_https_port = 12346 +preserving_data_port = 12347 + +queries = [ + "select *, column1*column2*column3 from file('{}', 'CSV', '{}')".format(os.path.expanduser('~/test.csv'), format), + "select *, column1*column2*column3 from url('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format), + "select *, column1*column2*column3 from s3('http://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format), + "select *, column1*column2*column3 from s3('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format), + "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_http_port, format), + "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_https_port, format), +] + +put_queries = [ + "insert into table function s3('http://{}:{}/', 'CSV', '{}') values {}" + .format(redirecting_host, preserving_data_port, format, values), +] + + +class RedirectingToHTTPHTTPServer(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(307) + self.send_header('Content-type', 'text/xml') + self.send_header('Location', 'http://storage.yandexcloud.net/milovidov/test.csv') + self.end_headers() + self.wfile.write(bytes(r''' + + TemporaryRedirect + Please re-send this request to the specified temporary endpoint. + Continue to use the original request endpoint for future requests. + johnsmith.s3-gztb4pa9sq.amazonaws.com +''', "utf-8")) + + +class RedirectingToHTTPSHTTPServer(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(307) + self.send_header('Content-type', 'text/xml') + self.send_header('Location', 'https://storage.yandexcloud.net/milovidov/test.csv') + self.end_headers() + self.wfile.write(bytes(r''' + + TemporaryRedirect + Please re-send this request to the specified temporary endpoint. + Continue to use the original request endpoint for future requests. + johnsmith.s3-gztb4pa9sq.amazonaws.com +''', "utf-8")) + + +received_data = [] + + +class PreservingDataServer(http.server.BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + received_data.append(self.rfile.read()) + self.wfile.flush() + + +servers = [] +def redirecting_to_https_thread(): + server = http.server.HTTPServer((redirecting_host, redirecting_to_https_port), RedirectingToHTTPSHTTPServer) + servers.append(server) + server.handle_request() + +def redirecting_to_http_thread(): + server = http.server.HTTPServer((redirecting_host, redirecting_to_http_port), RedirectingToHTTPHTTPServer) + servers.append(server) + server.handle_request() + +def preserving_thread(): + server = http.server.HTTPServer((redirecting_host, preserving_data_port), PreservingDataServer) + servers.append(server) + server.handle_request() + + +jobs = [] +jobs.append(threading.Thread(target=redirecting_to_http_thread)) +jobs.append(threading.Thread(target=redirecting_to_https_thread)) +jobs.append(threading.Thread(target=preserving_thread)) +[ job.start() for job in jobs ] + +for query in queries: + print(query) + result = subprocess.run([ + os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'), + '-c', + os.path.expanduser('~/config.xml'), + '-q', + query + ], stdout=subprocess.PIPE, universal_newlines=True) + result.check_returncode() + unittest.TestCase().assertEqual(list(map(str.split, result.stdout.splitlines())), [ + ['1', '2', '3', '6'], + ['3', '2', '1', '6'], + ['78', '43', '45', '150930'], + ]) + +for query in put_queries: + print(query) + result = subprocess.run([ + os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'), + '-c', + os.path.expanduser('~/config.xml'), + '-q', + query + ], stdout=subprocess.PIPE, universal_newlines=True) + result.check_returncode() + unittest.TestCase().assertEqual(received_data[-1].decode(), '15\r\n1,2,3\n2,3,1\n78,43,45\n\r\n0\r\n\r\n') + +[ server.socket.close() for server in servers ] +[ job.join() for job in jobs ] diff --git a/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/config.xml b/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/config.xml new file mode 100644 index 00000000000..de0dd0a7087 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00950_table_function_s3_wip/config.xml @@ -0,0 +1,115 @@ + + default + + trace + /home/excitoon/clickhouse-server.log + /home/excitoon/clickhouse-server.err.log + 1000M + 10 + + + + + + 8 + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + a = 1 + + + + + a + b < 1 or c - d > 5 + + + + + c = 1 + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + +