tests (WIP)

dbms/tests/queries/0_stateless/00950_table_function_s3_wip/clickhouse-test

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import http.server
+import os
+import subprocess
+import threading
+import unittest
+
+
+format = 'column1 UInt32, column2 UInt32, column3 UInt32'
+values = '(1, 2, 3), (2, 3, 1), (78, 43, 45)'
+redirecting_host = '127.0.0.1'
+redirecting_to_http_port = 12345
+redirecting_to_https_port = 12346
+preserving_data_port = 12347
+
+queries = [
+    "select *, column1*column2*column3 from file('{}', 'CSV', '{}')".format(os.path.expanduser('~/test.csv'), format),
+    "select *, column1*column2*column3 from url('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
+    "select *, column1*column2*column3 from s3('http://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
+    "select *, column1*column2*column3 from s3('https://storage.yandexcloud.net/milovidov/test.csv', 'CSV', '{}')".format(format),
+    "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_http_port, format),
+    "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(redirecting_host, redirecting_to_https_port, format),
+]
+
+put_queries = [
+    "insert into table function s3('http://{}:{}/', 'CSV', '{}') values {}"
+        .format(redirecting_host, preserving_data_port, format, values),
+]
+
+
+class RedirectingToHTTPHTTPServer(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(307)
+        self.send_header('Content-type', 'text/xml')
+        self.send_header('Location', 'http://storage.yandexcloud.net/milovidov/test.csv')
+        self.end_headers()
+        self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>TemporaryRedirect</Code>
+  <Message>Please re-send this request to the specified temporary endpoint.
+  Continue to use the original request endpoint for future requests.</Message>
+  <Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
+</Error>''', "utf-8"))
+
+
+class RedirectingToHTTPSHTTPServer(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(307)
+        self.send_header('Content-type', 'text/xml')
+        self.send_header('Location', 'https://storage.yandexcloud.net/milovidov/test.csv')
+        self.end_headers()
+        self.wfile.write(bytes(r'''<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>TemporaryRedirect</Code>
+  <Message>Please re-send this request to the specified temporary endpoint.
+  Continue to use the original request endpoint for future requests.</Message>
+  <Endpoint>johnsmith.s3-gztb4pa9sq.amazonaws.com</Endpoint>
+</Error>''', "utf-8"))
+
+
+received_data = []
+
+
+class PreservingDataServer(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/plain')
+        self.end_headers()
+        received_data.append(self.rfile.read())
+        self.wfile.flush()
+
+
+servers = []
+def redirecting_to_https_thread():
+    server = http.server.HTTPServer((redirecting_host, redirecting_to_https_port), RedirectingToHTTPSHTTPServer)
+    servers.append(server)
+    server.handle_request()
+
+def redirecting_to_http_thread():
+    server = http.server.HTTPServer((redirecting_host, redirecting_to_http_port), RedirectingToHTTPHTTPServer)
+    servers.append(server)
+    server.handle_request()
+
+def preserving_thread():
+    server = http.server.HTTPServer((redirecting_host, preserving_data_port), PreservingDataServer)
+    servers.append(server)
+    server.handle_request()
+
+
+jobs = []
+jobs.append(threading.Thread(target=redirecting_to_http_thread))
+jobs.append(threading.Thread(target=redirecting_to_https_thread))
+jobs.append(threading.Thread(target=preserving_thread))
+[ job.start() for job in jobs ]
+
+for query in queries:
+    print(query)
+    result = subprocess.run([
+        os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
+        '-c',
+        os.path.expanduser('~/config.xml'),
+        '-q',
+        query
+    ], stdout=subprocess.PIPE, universal_newlines=True)
+    result.check_returncode()
+    unittest.TestCase().assertEqual(list(map(str.split, result.stdout.splitlines())), [
+        ['1', '2', '3', '6'],
+        ['3', '2', '1', '6'],
+        ['78', '43', '45', '150930'],
+    ])
+
+for query in put_queries:
+    print(query)
+    result = subprocess.run([
+        os.path.expanduser('~/ClickHouse-bin/dbms/programs/clickhouse-local'),
+        '-c',
+        os.path.expanduser('~/config.xml'),
+        '-q',
+        query
+    ], stdout=subprocess.PIPE, universal_newlines=True)
+    result.check_returncode()
+    unittest.TestCase().assertEqual(received_data[-1].decode(), '15\r\n1,2,3\n2,3,1\n78,43,45\n\r\n0\r\n\r\n')
+
+[ server.socket.close() for server in servers ]
+[ job.join() for job in jobs ]

dbms/tests/queries/0_stateless/00950_table_function_s3_wip/config.xml

@@ -0,0 +1,115 @@
+<config>
+    <default_profile>default</default_profile>
+    <logger>
+        <level>trace</level>
+        <log>/home/excitoon/clickhouse-server.log</log>
+        <errorlog>/home/excitoon/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>10</count>
+    </logger>
+<profiles>
+    <!-- Default settings -->
+    <default>
+        <!-- The maximum number of threads when running a single query. -->
+        <max_threads>8</max_threads>
+    </default>
+</profiles>
+    <users>
+        <!-- If user name was not specified, 'default' user is used. -->
+        <default>
+            <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+
+                 If you want to specify password in plaintext (not recommended), place it in 'password' element.
+                 Example: <password>qwerty</password>.
+                 Password could be empty.
+
+                 If you want to specify SHA256, place it in 'password_sha256_hex' element.
+                 Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+
+                 How to generate decent password:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+                 In first line will be password and in second - corresponding SHA256.
+            -->
+            <password></password>
+
+            <!-- List of networks with open access.
+
+                 To open access from everywhere, specify:
+                    <ip>::/0</ip>
+
+                 To open access only from localhost, specify:
+                    <ip>::1</ip>
+                    <ip>127.0.0.1</ip>
+
+                 Each element of list has one of the following forms:
+                 <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+                     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+                 <host> Hostname. Example: server01.yandex.ru.
+                     To check access, DNS query is performed, and all received addresses compared to peer address.
+                 <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
+                     To check access, DNS PTR query is performed for peer address and then regexp is applied.
+                     Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+                     Strongly recommended that regexp is ends with $
+                 All results of DNS requests are cached till server restart.
+            -->
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <!-- Settings profile for user. -->
+            <profile>default</profile>
+
+            <!-- Quota for user. -->
+            <quota>default</quota>
+
+            <!-- For testing the table filters -->
+            <databases>
+                <test>
+                    <!-- Simple expression filter -->
+                    <filtered_table1>
+                        <filter>a = 1</filter>
+                    </filtered_table1>
+
+                    <!-- Complex expression filter -->
+                    <filtered_table2>
+                        <filter>a + b &lt; 1 or c - d &gt; 5</filter>
+                    </filtered_table2>
+
+                    <!-- Filter with ALIAS column -->
+                    <filtered_table3>
+                        <filter>c = 1</filter>
+                    </filtered_table3>
+                </test>
+            </databases>
+        </default>
+
+        <!-- Example of user with readonly access. -->
+        <!-- <readonly>
+            <password></password>
+            <networks incl="networks" replace="replace">
+                <ip>::1</ip>
+                <ip>127.0.0.1</ip>
+            </networks>
+            <profile>readonly</profile>
+            <quota>default</quota>
+        </readonly> -->
+    </users>
+    <!-- Quotas. -->
+    <quotas>
+        <!-- Name of quota. -->
+        <default>
+            <!-- Limits for time interval. You could specify many intervals with different limits. -->
+            <interval>
+                <!-- Length of interval. -->
+                <duration>3600</duration>
+
+                <!-- No limits. Just calculate resource usage for time interval. -->
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</config>