tests (WIP)

This commit is contained in:
Vladimir Chebotarev 2019-05-31 18:16:40 +00:00
parent 062db0ec14
commit 52f242daf0
2 changed files with 241 additions and 0 deletions

View File

@ -0,0 +1,126 @@
#!/usr/bin/env python3
import http.server
import os
import subprocess
import threading
import unittest
format = 'column1 UInt32, column2 UInt32, column3 UInt32'
values = '(1, 2, 3), (2, 3, 1), (78, 43, 45)'
redirecting_host = '127.0.0.1'
redirecting_to_http_port = 12345
redirecting_to_https_port = 12346
preserving_data_port = 12347
queries = [
"select *, column1*column2*column3 from file('{}', 'CSV', '{}')".format(os.path.expanduser('~/test.csv'), format),
"select *, column1*column2*column3 from url('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
"select *, column1*column2*column3 from s3('http://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
"select *, column1*column2*column3 from s3('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_http_port, format),
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_https_port, format),
]
put_queries = [
"insert into table function s3('http://{}:{}/', 'CSV', '{}') values {}"
.format(redirecting_host, preserving_data_port, format, values),
]
class RedirectingToHTTPHTTPServer(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(307)
self.send_header('Content-type', 'text/xml')
self.send_header('Location', 'http://storage.yandexcloud.net/milovidov/test.csv')
self.end_headers()
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>TemporaryRedirect</Code>
<Message>Please re-send this request to the specified temporary endpoint.
Continue to use the original request endpoint for future requests.</Message>
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
</Error>''', "utf-8"))
class RedirectingToHTTPSHTTPServer(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(307)
self.send_header('Content-type', 'text/xml')
self.send_header('Location', 'https://storage.yandexcloud.net/milovidov/test.csv')
self.end_headers()
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>TemporaryRedirect</Code>
<Message>Please re-send this request to the specified temporary endpoint.
Continue to use the original request endpoint for future requests.</Message>
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
</Error>''', "utf-8"))
received_data = []
class PreservingDataServer(http.server.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
received_data.append(self.rfile.read())
self.wfile.flush()
servers = []
def redirecting_to_https_thread():
server = http.server.HTTPServer((redirecting_host, redirecting_to_https_port), RedirectingToHTTPSHTTPServer)
servers.append(server)
server.handle_request()
def redirecting_to_http_thread():
server = http.server.HTTPServer((redirecting_host, redirecting_to_http_port), RedirectingToHTTPHTTPServer)
servers.append(server)
server.handle_request()
def preserving_thread():
server = http.server.HTTPServer((redirecting_host, preserving_data_port), PreservingDataServer)
servers.append(server)
server.handle_request()
jobs = []
jobs.append(threading.Thread(target=redirecting_to_http_thread))
jobs.append(threading.Thread(target=redirecting_to_https_thread))
jobs.append(threading.Thread(target=preserving_thread))
[ job.start() for job in jobs ]
for query in queries:
print(query)
result = subprocess.run([
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
'-c',
os.path.expanduser('~/config.xml'),
'-q',
query
], stdout=subprocess.PIPE, universal_newlines=True)
result.check_returncode()
unittest.TestCase().assertEqual(list(map(str.split, result.stdout.splitlines())), [
['1', '2', '3', '6'],
['3', '2', '1', '6'],
['78', '43', '45', '150930'],
])
for query in put_queries:
print(query)
result = subprocess.run([
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
'-c',
os.path.expanduser('~/config.xml'),
'-q',
query
], stdout=subprocess.PIPE, universal_newlines=True)
result.check_returncode()
unittest.TestCase().assertEqual(received_data[-1].decode(), '15\r\n1,2,3\n2,3,1\n78,43,45\n\r\n0\r\n\r\n')
[ server.socket.close() for server in servers ]
[ job.join() for job in jobs ]

View File

@ -0,0 +1,115 @@
<config>
<default_profile>default</default_profile>
<logger>
<level>trace</level>
<log>/home/excitoon/clickhouse-server.log</log>
<errorlog>/home/excitoon/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>10</count>
</logger>
<profiles>
<!-- Default settings -->
<default>
<!-- The maximum number of threads when running a single query. -->
<max_threads>8</max_threads>
</default>
</profiles>
<users>
<!-- If user name was not specified, 'default' user is used. -->
<default>
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
If you want to specify password in plaintext (not recommended), place it in 'password' element.
Example: <password>qwerty</password>.
Password could be empty.
If you want to specify SHA256, place it in 'password_sha256_hex' element.
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
How to generate decent password:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
In first line will be password and in second - corresponding SHA256.
-->
<password></password>
<!-- List of networks with open access.
To open access from everywhere, specify:
<ip>::/0</ip>
To open access only from localhost, specify:
<ip>::1</ip>
<ip>127.0.0.1</ip>
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
To check access, DNS PTR query is performed for peer address and then regexp is applied.
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
Strongly recommended that regexp is ends with $
All results of DNS requests are cached till server restart.
-->
<networks>
<ip>::/0</ip>
</networks>
<!-- Settings profile for user. -->
<profile>default</profile>
<!-- Quota for user. -->
<quota>default</quota>
<!-- For testing the table filters -->
<databases>
<test>
<!-- Simple expression filter -->
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<!-- Complex expression filter -->
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
<!-- Filter with ALIAS column -->
<filtered_table3>
<filter>c = 1</filter>
</filtered_table3>
</test>
</databases>
</default>
<!-- Example of user with readonly access. -->
<!-- <readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly> -->
</users>
<!-- Quotas. -->
<quotas>
<!-- Name of quota. -->
<default>
<!-- Limits for time interval. You could specify many intervals with different limits. -->
<interval>
<!-- Length of interval. -->
<duration>3600</duration>
<!-- No limits. Just calculate resource usage for time interval. -->
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</config>