ClickHouse/tests/integration/test_globs_in_filepath/test.py

import pytest

from helpers.cluster import ClickHouseCluster

cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node')
path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/"   # should be the same as in config file

@pytest.fixture(scope="module")
def start_cluster():
    try:
        cluster.start()

        yield cluster

    except Exception as ex:
        print(ex)
        raise ex
    finally:
        cluster.shutdown()

def test_strange_filenames(start_cluster):
    # 2 rows data
    some_data = "\t111.222\nData\t333.444"

    node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], privileged=True, user='root')

    files = ["p.o.i.n.t.s",
             "b}{ra{ces",
             "b}.o{t.h"]

    # filename inside testing data for debug simplicity
    for filename in files:
        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')

    test_requests = [("p.o.??n.t.s", "2"),
                     ("p.o.*t.s", "2"),
                     ("b}{r?{ces", "2"),
                     ("b}*ces", "2"),
                     ("b}.?{t.h", "2")]

    for pattern, value in test_requests:
        assert node.query('''
            select count(*) from file('strange_names/{}', 'TSV', 'text String, number Float64')
        '''.format(pattern)) == '{}\n'.format(value)
        assert node.query('''
            select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64')
        '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)

def test_linear_structure(start_cluster):
    # 2 rows data
    some_data = "\t123.456\nData\t789.012"

    files = ["file1", "file2", "file3", "file4", "file5",
             "file000", "file111", "file222", "file333", "file444",
             "a_file", "b_file", "c_file", "d_file", "e_file",
             "a_data", "b_data", "c_data", "d_data", "e_data"]

    # filename inside testing data for debug simplicity
    for filename in files:
        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')

    test_requests = [("file{0..9}", "10"),
                     ("file?", "10"),
                     ("nothing*", "0"),
                     ("file{0..9}{0..9}{0..9}", "10"),
                     ("file{000..999}", "10"),
                     ("file???", "10"),
                     ("file*", "20"),
                     ("a_{file,data}", "4"),
                     ("?_{file,data}", "20"),
                     ("{a,b,c,d,e}_{file,data}", "20"),
                     ("{a,b,c,d,e}?{file,data}", "20"),
                     ("*", "40")]

    for pattern, value in test_requests:
        assert node.query('''
            select count(*) from file('{}', 'TSV', 'text String, number Float64')
        '''.format(pattern)) == '{}\n'.format(value)
        assert node.query('''
            select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
        '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)

def test_deep_structure(start_cluster):
    # 2 rows data
    some_data = "\t135.791\nData\t246.802"
    dirs = ["directory1/", "directory2/", "some_more_dir/", "we/",
            "directory1/big_dir/",
            "directory1/dir1/", "directory1/dir2/", "directory1/dir3/",
            "directory2/dir1/", "directory2/dir2/", "directory2/one_more_dir/",
            "some_more_dir/yet_another_dir/",
            "we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"]

    for dir in dirs:
        node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], privileged=True, user='root')

    # all directories appeared in files must be listed in dirs
    files = []
    for i in range(10):
        for j in range(10):
            for k in range(10):
                files.append("directory1/big_dir/file" + str(i) + str(j) + str(k))

    for dir in dirs:
        files.append(dir+"file")

    # filename inside testing data for debug simplicity
    for filename in files:
        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')

    test_requests = [ ("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"),
                     ("?", "0"),
                     ("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"),
                     ("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")]

    for pattern, value in test_requests:
        assert node.query('''
            select count(*) from file('{}', 'TSV', 'text String, number Float64')
        '''.format(pattern)) == '{}\n'.format(value)
        assert node.query('''
            select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
        '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)

def test_table_function_and_virtual_columns(start_cluster):
    node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)])
    node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)])
    node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)")
    assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000'
    node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)")
    assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1'
    assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
    assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
    assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
    assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()