mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 09:32:01 +00:00
tests (WIP)
This commit is contained in:
Vladimir Chebotarev
parent
062db0ec14
commit
52f242daf0
126
dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test
Executable file
126
dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test
Executable file
@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import http.server
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
|
||||
format = 'column1 UInt32, column2 UInt32, column3 UInt32'
|
||||
values = '(1, 2, 3), (2, 3, 1), (78, 43, 45)'
|
||||
redirecting_host = '127.0.0.1'
|
||||
redirecting_to_http_port = 12345
|
||||
redirecting_to_https_port = 12346
|
||||
preserving_data_port = 12347
|
||||
|
||||
queries = [
|
||||
"select *, column1*column2*column3 from file('{}', 'CSV', '{}')".format(os.path.expanduser('~/test.csv'), format),
|
||||
"select *, column1*column2*column3 from url('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||
"select *, column1*column2*column3 from s3('http://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||
"select *, column1*column2*column3 from s3('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
|
||||
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_http_port, format),
|
||||
"select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_https_port, format),
|
||||
]
|
||||
|
||||
put_queries = [
|
||||
"insert into table function s3('http://{}:{}/', 'CSV', '{}') values {}"
|
||||
.format(redirecting_host, preserving_data_port, format, values),
|
||||
]
|
||||
|
||||
|
||||
class RedirectingToHTTPHTTPServer(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(307)
|
||||
self.send_header('Content-type', 'text/xml')
|
||||
self.send_header('Location', 'http://storage.yandexcloud.net/milovidov/test.csv')
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>TemporaryRedirect</Code>
|
||||
<Message>Please re-send this request to the specified temporary endpoint.
|
||||
Continue to use the original request endpoint for future requests.</Message>
|
||||
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
|
||||
</Error>''', "utf-8"))
|
||||
|
||||
|
||||
class RedirectingToHTTPSHTTPServer(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(307)
|
||||
self.send_header('Content-type', 'text/xml')
|
||||
self.send_header('Location', 'https://storage.yandexcloud.net/milovidov/test.csv')
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>TemporaryRedirect</Code>
|
||||
<Message>Please re-send this request to the specified temporary endpoint.
|
||||
Continue to use the original request endpoint for future requests.</Message>
|
||||
<Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
|
||||
</Error>''', "utf-8"))
|
||||
|
||||
|
||||
received_data = []
|
||||
|
||||
|
||||
class PreservingDataServer(http.server.BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-type', 'text/plain')
|
||||
self.end_headers()
|
||||
received_data.append(self.rfile.read())
|
||||
self.wfile.flush()
|
||||
|
||||
|
||||
servers = []
|
||||
def redirecting_to_https_thread():
|
||||
server = http.server.HTTPServer((redirecting_host, redirecting_to_https_port), RedirectingToHTTPSHTTPServer)
|
||||
servers.append(server)
|
||||
server.handle_request()
|
||||
|
||||
def redirecting_to_http_thread():
|
||||
server = http.server.HTTPServer((redirecting_host, redirecting_to_http_port), RedirectingToHTTPHTTPServer)
|
||||
servers.append(server)
|
||||
server.handle_request()
|
||||
|
||||
def preserving_thread():
|
||||
server = http.server.HTTPServer((redirecting_host, preserving_data_port), PreservingDataServer)
|
||||
servers.append(server)
|
||||
server.handle_request()
|
||||
|
||||
|
||||
jobs = []
|
||||
jobs.append(threading.Thread(target=redirecting_to_http_thread))
|
||||
jobs.append(threading.Thread(target=redirecting_to_https_thread))
|
||||
jobs.append(threading.Thread(target=preserving_thread))
|
||||
[ job.start() for job in jobs ]
|
||||
|
||||
for query in queries:
|
||||
print(query)
|
||||
result = subprocess.run([
|
||||
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
|
||||
'-c',
|
||||
os.path.expanduser('~/config.xml'),
|
||||
'-q',
|
||||
query
|
||||
], stdout=subprocess.PIPE, universal_newlines=True)
|
||||
result.check_returncode()
|
||||
unittest.TestCase().assertEqual(list(map(str.split, result.stdout.splitlines())), [
|
||||
['1', '2', '3', '6'],
|
||||
['3', '2', '1', '6'],
|
||||
['78', '43', '45', '150930'],
|
||||
])
|
||||
|
||||
for query in put_queries:
|
||||
print(query)
|
||||
result = subprocess.run([
|
||||
os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
|
||||
'-c',
|
||||
os.path.expanduser('~/config.xml'),
|
||||
'-q',
|
||||
query
|
||||
], stdout=subprocess.PIPE, universal_newlines=True)
|
||||
result.check_returncode()
|
||||
unittest.TestCase().assertEqual(received_data[-1].decode(), '15\r\n1,2,3\n2,3,1\n78,43,45\n\r\n0\r\n\r\n')
|
||||
|
||||
[ server.socket.close() for server in servers ]
|
||||
[ job.join() for job in jobs ]
|
||||
@ -0,0 +1,115 @@
|
||||
<config>
|
||||
<default_profile>default</default_profile>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<log>/home/excitoon/clickhouse-server.log</log>
|
||||
<errorlog>/home/excitoon/clickhouse-server.err.log</errorlog>
|
||||
<size>1000M</size>
|
||||
<count>10</count>
|
||||
</logger>
|
||||
<profiles>
|
||||
<!-- Default settings -->
|
||||
<default>
|
||||
<!-- The maximum number of threads when running a single query. -->
|
||||
<max_threads>8</max_threads>
|
||||
</default>
|
||||
</profiles>
|
||||
<users>
|
||||
<!-- If user name was not specified, 'default' user is used. -->
|
||||
<default>
|
||||
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
|
||||
|
||||
If you want to specify password in plaintext (not recommended), place it in 'password' element.
|
||||
Example: <password>qwerty</password>.
|
||||
Password could be empty.
|
||||
|
||||
If you want to specify SHA256, place it in 'password_sha256_hex' element.
|
||||
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
|
||||
|
||||
How to generate decent password:
|
||||
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
|
||||
In first line will be password and in second - corresponding SHA256.
|
||||
-->
|
||||
<password></password>
|
||||
|
||||
<!-- List of networks with open access.
|
||||
|
||||
To open access from everywhere, specify:
|
||||
<ip>::/0</ip>
|
||||
|
||||
To open access only from localhost, specify:
|
||||
<ip>::1</ip>
|
||||
<ip>127.0.0.1</ip>
|
||||
|
||||
Each element of list has one of the following forms:
|
||||
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
|
||||
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||
<host> Hostname. Example: server01.yandex.ru.
|
||||
To check access, DNS query is performed, and all received addresses compared to peer address.
|
||||
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
|
||||
To check access, DNS PTR query is performed for peer address and then regexp is applied.
|
||||
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
|
||||
Strongly recommended that regexp is ends with $
|
||||
All results of DNS requests are cached till server restart.
|
||||
-->
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<!-- Settings profile for user. -->
|
||||
<profile>default</profile>
|
||||
|
||||
<!-- Quota for user. -->
|
||||
<quota>default</quota>
|
||||
|
||||
<!-- For testing the table filters -->
|
||||
<databases>
|
||||
<test>
|
||||
<!-- Simple expression filter -->
|
||||
<filtered_table1>
|
||||
<filter>a = 1</filter>
|
||||
</filtered_table1>
|
||||
|
||||
<!-- Complex expression filter -->
|
||||
<filtered_table2>
|
||||
<filter>a + b < 1 or c - d > 5</filter>
|
||||
</filtered_table2>
|
||||
|
||||
<!-- Filter with ALIAS column -->
|
||||
<filtered_table3>
|
||||
<filter>c = 1</filter>
|
||||
</filtered_table3>
|
||||
</test>
|
||||
</databases>
|
||||
</default>
|
||||
|
||||
<!-- Example of user with readonly access. -->
|
||||
<!-- <readonly>
|
||||
<password></password>
|
||||
<networks incl="networks" replace="replace">
|
||||
<ip>::1</ip>
|
||||
<ip>127.0.0.1</ip>
|
||||
</networks>
|
||||
<profile>readonly</profile>
|
||||
<quota>default</quota>
|
||||
</readonly> -->
|
||||
</users>
|
||||
<!-- Quotas. -->
|
||||
<quotas>
|
||||
<!-- Name of quota. -->
|
||||
<default>
|
||||
<!-- Limits for time interval. You could specify many intervals with different limits. -->
|
||||
<interval>
|
||||
<!-- Length of interval. -->
|
||||
<duration>3600</duration>
|
||||
|
||||
<!-- No limits. Just calculate resource usage for time interval. -->
|
||||
<queries>0</queries>
|
||||
<errors>0</errors>
|
||||
<result_rows>0</result_rows>
|
||||
<read_rows>0</read_rows>
|
||||
<execution_time>0</execution_time>
|
||||
</interval>
|
||||
</default>
|
||||
</quotas>
|
||||
</config>
|
||||
Loading…
Reference in New Issue
Block a user