mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-29 19:12:03 +00:00
tests (WIP)
This commit is contained in:
Vladimir Chebotarev
parent
062db0ec14
commit
52f242daf0
126
dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test
Executable file
126
dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test
Executable file
@ -0,0 +1,126 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import http.server
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
format = 'column1 UInt32, column2 UInt32, column3 UInt32'
|
||||||
|
values = '(1, 2, 3), (2, 3, 1), (78, 43, 45)'
|
||||||
|
redirecting_host = '127.0.0.1'
|
||||||
|
redirecting_to_http_port = 12345
|
||||||
|
redirecting_to_https_port = 12346
|
||||||
|
preserving_data_port = 12347
|
||||||
|
|
||||||
|
queries = [
|
||||||
|
"select *, column1*column2*column3 from file('{}', 'CSV', '{}')".format(os.path.expanduser('~/test.csv'), format),
|
||||||
|
"select *, column1*column2*column3 from url('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||||
|
"select *, column1*column2*column3 from s3('http://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||||
|
"select *, column1*column2*column3 from s3('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||||
|
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_http_port, format),
|
||||||
|
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_https_port, format),
|
||||||
|
]
|
||||||
|
|
||||||
|
put_queries = [
|
||||||
|
"insert into table function s3('http://{}:{}/', 'CSV', '{}') values {}"
|
||||||
|
.format(redirecting_host, preserving_data_port, format, values),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class RedirectingToHTTPHTTPServer(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.send_response(307)
|
||||||
|
self.send_header('Content-type', 'text/xml')
|
||||||
|
self.send_header('Location', 'http://storage.yandexcloud.net/milovidov/test.csv')
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<Error>
|
||||||
|
<Code>TemporaryRedirect</Code>
|
||||||
|
<Message>Please re-send this request to the specified temporary endpoint.
|
||||||
|
Continue to use the original request endpoint for future requests.</Message>
|
||||||
|
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
|
||||||
|
</Error>''', "utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
class RedirectingToHTTPSHTTPServer(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.send_response(307)
|
||||||
|
self.send_header('Content-type', 'text/xml')
|
||||||
|
self.send_header('Location', 'https://storage.yandexcloud.net/milovidov/test.csv')
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<Error>
|
||||||
|
<Code>TemporaryRedirect</Code>
|
||||||
|
<Message>Please re-send this request to the specified temporary endpoint.
|
||||||
|
Continue to use the original request endpoint for future requests.</Message>
|
||||||
|
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
|
||||||
|
</Error>''', "utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
received_data = []
|
||||||
|
|
||||||
|
|
||||||
|
class PreservingDataServer(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_POST(self):
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header('Content-type', 'text/plain')
|
||||||
|
self.end_headers()
|
||||||
|
received_data.append(self.rfile.read())
|
||||||
|
self.wfile.flush()
|
||||||
|
|
||||||
|
|
||||||
|
servers = []
|
||||||
|
def redirecting_to_https_thread():
|
||||||
|
server = http.server.HTTPServer((redirecting_host, redirecting_to_https_port), RedirectingToHTTPSHTTPServer)
|
||||||
|
servers.append(server)
|
||||||
|
server.handle_request()
|
||||||
|
|
||||||
|
def redirecting_to_http_thread():
|
||||||
|
server = http.server.HTTPServer((redirecting_host, redirecting_to_http_port), RedirectingToHTTPHTTPServer)
|
||||||
|
servers.append(server)
|
||||||
|
server.handle_request()
|
||||||
|
|
||||||
|
def preserving_thread():
|
||||||
|
server = http.server.HTTPServer((redirecting_host, preserving_data_port), PreservingDataServer)
|
||||||
|
servers.append(server)
|
||||||
|
server.handle_request()
|
||||||
|
|
||||||
|
|
||||||
|
jobs = []
|
||||||
|
jobs.append(threading.Thread(target=redirecting_to_http_thread))
|
||||||
|
jobs.append(threading.Thread(target=redirecting_to_https_thread))
|
||||||
|
jobs.append(threading.Thread(target=preserving_thread))
|
||||||
|
[ job.start() for job in jobs ]
|
||||||
|
|
||||||
|
for query in queries:
|
||||||
|
print(query)
|
||||||
|
result = subprocess.run([
|
||||||
|
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
|
||||||
|
'-c',
|
||||||
|
os.path.expanduser('~/config.xml'),
|
||||||
|
'-q',
|
||||||
|
query
|
||||||
|
], stdout=subprocess.PIPE, universal_newlines=True)
|
||||||
|
result.check_returncode()
|
||||||
|
unittest.TestCase().assertEqual(list(map(str.split, result.stdout.splitlines())), [
|
||||||
|
['1', '2', '3', '6'],
|
||||||
|
['3', '2', '1', '6'],
|
||||||
|
['78', '43', '45', '150930'],
|
||||||
|
])
|
||||||
|
|
||||||
|
for query in put_queries:
|
||||||
|
print(query)
|
||||||
|
result = subprocess.run([
|
||||||
|
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
|
||||||
|
'-c',
|
||||||
|
os.path.expanduser('~/config.xml'),
|
||||||
|
'-q',
|
||||||
|
query
|
||||||
|
], stdout=subprocess.PIPE, universal_newlines=True)
|
||||||
|
result.check_returncode()
|
||||||
|
unittest.TestCase().assertEqual(received_data[-1].decode(), '15\r\n1,2,3\n2,3,1\n78,43,45\n\r\n0\r\n\r\n')
|
||||||
|
|
||||||
|
[ server.socket.close() for server in servers ]
|
||||||
|
[ job.join() for job in jobs ]
|
||||||
@ -0,0 +1,115 @@
|
|||||||
|
<config>
|
||||||
|
<default_profile>default</default_profile>
|
||||||
|
<logger>
|
||||||
|
<level>trace</level>
|
||||||
|
<log>/home/excitoon/clickhouse-server.log</log>
|
||||||
|
<errorlog>/home/excitoon/clickhouse-server.err.log</errorlog>
|
||||||
|
<size>1000M</size>
|
||||||
|
<count>10</count>
|
||||||
|
</logger>
|
||||||
|
<profiles>
|
||||||
|
<!-- Default settings -->
|
||||||
|
<default>
|
||||||
|
<!-- The maximum number of threads when running a single query. -->
|
||||||
|
<max_threads>8</max_threads>
|
||||||
|
</default>
|
||||||
|
</profiles>
|
||||||
|
<users>
|
||||||
|
<!-- If user name was not specified, 'default' user is used. -->
|
||||||
|
<default>
|
||||||
|
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
|
||||||
|
|
||||||
|
If you want to specify password in plaintext (not recommended), place it in 'password' element.
|
||||||
|
Example: <password>qwerty</password>.
|
||||||
|
Password could be empty.
|
||||||
|
|
||||||
|
If you want to specify SHA256, place it in 'password_sha256_hex' element.
|
||||||
|
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
|
||||||
|
|
||||||
|
How to generate decent password:
|
||||||
|
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
|
||||||
|
In first line will be password and in second - corresponding SHA256.
|
||||||
|
-->
|
||||||
|
<password></password>
|
||||||
|
|
||||||
|
<!-- List of networks with open access.
|
||||||
|
|
||||||
|
To open access from everywhere, specify:
|
||||||
|
<ip>::/0</ip>
|
||||||
|
|
||||||
|
To open access only from localhost, specify:
|
||||||
|
<ip>::1</ip>
|
||||||
|
<ip>127.0.0.1</ip>
|
||||||
|
|
||||||
|
Each element of list has one of the following forms:
|
||||||
|
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
|
||||||
|
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||||
|
<host> Hostname. Example: server01.yandex.ru.
|
||||||
|
To check access, DNS query is performed, and all received addresses compared to peer address.
|
||||||
|
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
|
||||||
|
To check access, DNS PTR query is performed for peer address and then regexp is applied.
|
||||||
|
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
|
||||||
|
Strongly recommended that regexp is ends with $
|
||||||
|
All results of DNS requests are cached till server restart.
|
||||||
|
-->
|
||||||
|
<networks>
|
||||||
|
<ip>::/0</ip>
|
||||||
|
</networks>
|
||||||
|
|
||||||
|
<!-- Settings profile for user. -->
|
||||||
|
<profile>default</profile>
|
||||||
|
|
||||||
|
<!-- Quota for user. -->
|
||||||
|
<quota>default</quota>
|
||||||
|
|
||||||
|
<!-- For testing the table filters -->
|
||||||
|
<databases>
|
||||||
|
<test>
|
||||||
|
<!-- Simple expression filter -->
|
||||||
|
<filtered_table1>
|
||||||
|
<filter>a = 1</filter>
|
||||||
|
</filtered_table1>
|
||||||
|
|
||||||
|
<!-- Complex expression filter -->
|
||||||
|
<filtered_table2>
|
||||||
|
<filter>a + b < 1 or c - d > 5</filter>
|
||||||
|
</filtered_table2>
|
||||||
|
|
||||||
|
<!-- Filter with ALIAS column -->
|
||||||
|
<filtered_table3>
|
||||||
|
<filter>c = 1</filter>
|
||||||
|
</filtered_table3>
|
||||||
|
</test>
|
||||||
|
</databases>
|
||||||
|
</default>
|
||||||
|
|
||||||
|
<!-- Example of user with readonly access. -->
|
||||||
|
<!-- <readonly>
|
||||||
|
<password></password>
|
||||||
|
<networks incl="networks" replace="replace">
|
||||||
|
<ip>::1</ip>
|
||||||
|
<ip>127.0.0.1</ip>
|
||||||
|
</networks>
|
||||||
|
<profile>readonly</profile>
|
||||||
|
<quota>default</quota>
|
||||||
|
</readonly> -->
|
||||||
|
</users>
|
||||||
|
<!-- Quotas. -->
|
||||||
|
<quotas>
|
||||||
|
<!-- Name of quota. -->
|
||||||
|
<default>
|
||||||
|
<!-- Limits for time interval. You could specify many intervals with different limits. -->
|
||||||
|
<interval>
|
||||||
|
<!-- Length of interval. -->
|
||||||
|
<duration>3600</duration>
|
||||||
|
|
||||||
|
<!-- No limits. Just calculate resource usage for time interval. -->
|
||||||
|
<queries>0</queries>
|
||||||
|
<errors>0</errors>
|
||||||
|
<result_rows>0</result_rows>
|
||||||
|
<read_rows>0</read_rows>
|
||||||
|
<execution_time>0</execution_time>
|
||||||
|
</interval>
|
||||||
|
</default>
|
||||||
|
</quotas>
|
||||||
|
</config>
|
||||||
Loading…
Reference in New Issue
Block a user