mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 16:50:48 +00:00
b75963d370
This PR formats all the `*.py` files found under the `tests/integration` folder. It also reorders the imports and cleans up a bunch of unused imports. The formatting also takes care of other things like wrapping lines and fixing spaces and indents such that the tests look more readable.
151 lines
6.8 KiB
Python
151 lines
6.8 KiB
Python
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node = cluster.add_instance('node')
|
|
path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
|
|
yield cluster
|
|
|
|
except Exception as ex:
|
|
print(ex)
|
|
raise ex
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_strange_filenames(start_cluster):
|
|
# 2 rows data
|
|
some_data = "\t111.222\nData\t333.444"
|
|
|
|
node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)],
|
|
privileged=True, user='root')
|
|
|
|
files = ["p.o.i.n.t.s",
|
|
"b}{ra{ces",
|
|
"b}.o{t.h"]
|
|
|
|
# filename inside testing data for debug simplicity
|
|
for filename in files:
|
|
node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data,
|
|
path_to_userfiles_from_defaut_config,
|
|
filename)], privileged=True,
|
|
user='root')
|
|
|
|
test_requests = [("p.o.??n.t.s", "2"),
|
|
("p.o.*t.s", "2"),
|
|
("b}{r?{ces", "2"),
|
|
("b}*ces", "2"),
|
|
("b}.?{t.h", "2")]
|
|
|
|
for pattern, value in test_requests:
|
|
assert node.query('''
|
|
select count(*) from file('strange_names/{}', 'TSV', 'text String, number Float64')
|
|
'''.format(pattern)) == '{}\n'.format(value)
|
|
assert node.query('''
|
|
select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64')
|
|
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
|
|
|
|
|
|
def test_linear_structure(start_cluster):
|
|
# 2 rows data
|
|
some_data = "\t123.456\nData\t789.012"
|
|
|
|
files = ["file1", "file2", "file3", "file4", "file5",
|
|
"file000", "file111", "file222", "file333", "file444",
|
|
"a_file", "b_file", "c_file", "d_file", "e_file",
|
|
"a_data", "b_data", "c_data", "d_data", "e_data"]
|
|
|
|
# filename inside testing data for debug simplicity
|
|
for filename in files:
|
|
node.exec_in_container(['bash', '-c',
|
|
'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
|
|
filename)], privileged=True, user='root')
|
|
|
|
test_requests = [("file{0..9}", "10"),
|
|
("file?", "10"),
|
|
("nothing*", "0"),
|
|
("file{0..9}{0..9}{0..9}", "10"),
|
|
("file{000..999}", "10"),
|
|
("file???", "10"),
|
|
("file*", "20"),
|
|
("a_{file,data}", "4"),
|
|
("?_{file,data}", "20"),
|
|
("{a,b,c,d,e}_{file,data}", "20"),
|
|
("{a,b,c,d,e}?{file,data}", "20"),
|
|
("*", "40")]
|
|
|
|
for pattern, value in test_requests:
|
|
assert node.query('''
|
|
select count(*) from file('{}', 'TSV', 'text String, number Float64')
|
|
'''.format(pattern)) == '{}\n'.format(value)
|
|
assert node.query('''
|
|
select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
|
|
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
|
|
|
|
|
|
def test_deep_structure(start_cluster):
|
|
# 2 rows data
|
|
some_data = "\t135.791\nData\t246.802"
|
|
dirs = ["directory1/", "directory2/", "some_more_dir/", "we/",
|
|
"directory1/big_dir/",
|
|
"directory1/dir1/", "directory1/dir2/", "directory1/dir3/",
|
|
"directory2/dir1/", "directory2/dir2/", "directory2/one_more_dir/",
|
|
"some_more_dir/yet_another_dir/",
|
|
"we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"]
|
|
|
|
for dir in dirs:
|
|
node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)],
|
|
privileged=True, user='root')
|
|
|
|
# all directories appeared in files must be listed in dirs
|
|
files = []
|
|
for i in range(10):
|
|
for j in range(10):
|
|
for k in range(10):
|
|
files.append("directory1/big_dir/file" + str(i) + str(j) + str(k))
|
|
|
|
for dir in dirs:
|
|
files.append(dir + "file")
|
|
|
|
# filename inside testing data for debug simplicity
|
|
for filename in files:
|
|
node.exec_in_container(['bash', '-c',
|
|
'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
|
|
filename)], privileged=True, user='root')
|
|
|
|
test_requests = [("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"),
|
|
("?", "0"),
|
|
("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"),
|
|
("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")]
|
|
|
|
for pattern, value in test_requests:
|
|
assert node.query('''
|
|
select count(*) from file('{}', 'TSV', 'text String, number Float64')
|
|
'''.format(pattern)) == '{}\n'.format(value)
|
|
assert node.query('''
|
|
select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
|
|
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
|
|
|
|
|
|
def test_table_function_and_virtual_columns(start_cluster):
|
|
node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)])
|
|
node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)])
|
|
node.query(
|
|
"insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)")
|
|
assert node.query(
|
|
"select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000'
|
|
node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)")
|
|
assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() == '1'
|
|
assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
|
|
assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
|
|
assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
|
|
assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
|