ClickHouse/tests/queries/query_test.py

import difflib
import os
import random
import string
import subprocess
import sys

import pytest


SKIP_LIST = [
    # these couple of tests hangs everything
    "00600_replace_running_query",
    "00987_distributed_stack_overflow",
    "01954_clickhouse_benchmark_multiple_long",

    # just fail
    "00133_long_shard_memory_tracker_and_exception_safety",
    "00463_long_sessions_in_http_interface",
    "00505_secure",
    "00505_shard_secure",
    "00646_url_engine",
    "00725_memory_tracking",  # BROKEN
    "00738_lock_for_inner_table",
    "00821_distributed_storage_with_join_on",
    "00825_protobuf_format_array_3dim",
    "00825_protobuf_format_array_of_arrays",
    "00825_protobuf_format_enum_mapping",
    "00825_protobuf_format_nested_in_nested",
    "00825_protobuf_format_nested_optional",
    "00825_protobuf_format_no_length_delimiter",
    "00825_protobuf_format_persons",
    "00825_protobuf_format_squares",
    "00825_protobuf_format_table_default",
    "00834_cancel_http_readonly_queries_on_client_close",
    "00877_memory_limit_for_new_delete",
    "00900_parquet_load",
    "00933_test_fix_extra_seek_on_compressed_cache",
    "00965_logs_level_bugfix",
    "00965_send_logs_level_concurrent_queries",
    "00974_query_profiler",
    "00990_hasToken",
    "00990_metric_log_table_not_empty",
    "01014_lazy_database_concurrent_recreate_reattach_and_show_tables",
    "01017_uniqCombined_memory_usage",
    "01018_Distributed__shard_num",
    "01018_ip_dictionary_long",
    "01035_lc_empty_part_bug",  # FLAKY
    "01050_clickhouse_dict_source_with_subquery",
    "01053_ssd_dictionary",
    "01054_cache_dictionary_overflow_cell",
    "01057_http_compression_prefer_brotli",
    "01080_check_for_error_incorrect_size_of_nested_column",
    "01083_expressions_in_engine_arguments",
    "01086_odbc_roundtrip",
    "01088_benchmark_query_id",
    "01092_memory_profiler",
    "01098_temporary_and_external_tables",
    "01099_parallel_distributed_insert_select",
    "01103_check_cpu_instructions_at_startup",
    "01107_atomic_db_detach_attach",
    "01114_database_atomic",
    "01148_zookeeper_path_macros_unfolding",
    "01152_cross_replication",  # tcp port in reference
    "01175_distributed_ddl_output_mode_long",
    "01181_db_atomic_drop_on_cluster",  # tcp port in reference
    "01280_ssd_complex_key_dictionary",
    "01293_client_interactive_vertical_multiline",  # expect-test
    "01293_client_interactive_vertical_singleline",  # expect-test
    "01293_show_clusters",
    "01293_show_settings",
    "01293_system_distribution_queue",  # FLAKY
    "01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long",
    "01294_system_distributed_on_cluster",
    "01300_client_save_history_when_terminated",  # expect-test
    "01304_direct_io",
    "01306_benchmark_json",
    "01035_lc_empty_part_bug",  # FLAKY
    "01175_distributed_ddl_output_mode_long",  # tcp port in reference
    "01320_create_sync_race_condition_zookeeper",
    "01355_CSV_input_format_allow_errors",
    "01370_client_autocomplete_word_break_characters",  # expect-test
    "01376_GROUP_BY_injective_elimination_dictGet",
    "01393_benchmark_secure_port",
    "01418_custom_settings",
    "01451_wrong_error_long_query",
    "01455_opentelemetry_distributed",
    "01473_event_time_microseconds",
    "01474_executable_dictionary",
    "01507_clickhouse_server_start_with_embedded_config",
    "01514_distributed_cancel_query_on_error",
    "01520_client_print_query_id",  # expect-test
    "01526_client_start_and_exit",  # expect-test
    "01526_max_untracked_memory",
    "01527_dist_sharding_key_dictGet_reload",
    "01528_play",
    "01545_url_file_format_settings",
    "01553_datetime64_comparison",
    "01555_system_distribution_queue_mask",
    "01558_ttest_scipy",
    "01561_mann_whitney_scipy",
    "01582_distinct_optimization",
    "01591_window_functions",
    "01594_too_low_memory_limits",
    "01599_multiline_input_and_singleline_comments",  # expect-test
    "01601_custom_tld",
    "01606_git_import",
    "01610_client_spawn_editor",  # expect-test
    "01654_test_writer_block_sequence",  # No module named 'pandas'
    "01658_read_file_to_stringcolumn",
    "01666_merge_tree_max_query_limit",
    "01674_unicode_asan",
    "01676_clickhouse_client_autocomplete",  # expect-test (partially)
    "01676_long_clickhouse_client_autocomplete",
    "01683_text_log_deadlock",  # secure tcp
    "01684_ssd_cache_dictionary_simple_key",
    "01685_ssd_cache_dictionary_complex_key",
    "01737_clickhouse_server_wait_server_pool_long",
    "01746_executable_pool_dictionary",
    "01747_executable_pool_dictionary_implicit_key.sql",
    "01747_join_view_filter_dictionary",
    "01748_dictionary_table_dot",
    "01754_cluster_all_replicas_shard_num",
    "01759_optimize_skip_unused_shards_zero_shards",
    "01763_max_distributed_depth",  # BROKEN
    "01780_clickhouse_dictionary_source_loop",
    "01801_s3_cluster",
    "01802_test_postgresql_protocol_with_row_policy",
    "01804_dictionary_decimal256_type",  # hardcoded path
    "01848_http_insert_segfault",
    "01875_ssd_cache_dictionary_decimal256_type",
    "01880_remote_ipv6",
    "01889_check_row_policy_defined_using_user_function",
    "01889_clickhouse_client_config_format",
    "01903_ssd_cache_dictionary_array_type",
]


def check_result(result, error, return_code, reference, replace_map):
    if replace_map:
        for old, new in replace_map.items():
            result = result.replace(old.encode('utf-8'), new.encode('utf-8'))

    if return_code != 0:
        try:
            print(error.decode('utf-8'), file=sys.stderr)
        except UnicodeDecodeError:
            print(error.decode('latin1'), file=sys.stderr)  # encoding with 1 symbol per 1 byte, covering all values
        pytest.fail('Client died unexpectedly with code {code}'.format(code=return_code), pytrace=False)
    elif result != reference:
        pytest.fail("Query output doesn't match reference:{eol}{diff}".format(
                eol=os.linesep,
                diff=os.linesep.join(l.strip() for l in difflib.unified_diff(reference.decode('utf-8').splitlines(),
                                                                             result.decode('utf-8').splitlines(),
                                                                             fromfile='expected', tofile='actual'))),
            pytrace=False)


def run_client(bin_prefix, port, database, query, reference, replace_map=None):
    # We can't use `text=True` since some tests may return binary data
    cmd = [bin_prefix + '-client', '--port', str(port), '-d', database, '-m', '-n', '--testmode']
    client = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    result, error = client.communicate(query.encode('utf-8'))
    assert client.returncode is not None, "Client should exit after processing all queries"

    check_result(result, error, client.returncode, reference, replace_map)


def run_shell(bin_prefix, server, database, path, reference, replace_map=None):
    env = {
        'CLICKHOUSE_BINARY': bin_prefix,
        'CLICKHOUSE_DATABASE': database,
        'CLICKHOUSE_PORT_TCP': str(server.tcp_port),
        'CLICKHOUSE_PORT_TCP_SECURE': str(server.tcps_port),
        'CLICKHOUSE_PORT_TCP_WITH_PROXY': str(server.proxy_port),
        'CLICKHOUSE_PORT_HTTP': str(server.http_port),
        'CLICKHOUSE_PORT_INTERSERVER': str(server.inter_port),
        'CLICKHOUSE_PORT_POSTGRESQL': str(server.postgresql_port),
        'CLICKHOUSE_TMP': server.tmp_dir,
        'CLICKHOUSE_CONFIG_CLIENT': server.client_config,
        'PROTOC_BINARY': os.path.abspath(os.path.join(os.path.dirname(bin_prefix), '..', 'contrib', 'protobuf', 'protoc')),  # FIXME: adhoc solution
    }
    shell = subprocess.Popen([path], env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    result, error = shell.communicate()
    assert shell.returncode is not None, "Script should exit after executing all commands"

    check_result(result, error, shell.returncode, reference, replace_map)


def random_str(length=10):
    alphabet = string.ascii_lowercase + string.digits
    random.seed(os.urandom(8))
    return ''.join(random.choice(alphabet) for _ in range(length))


def test_sql_query(bin_prefix, sql_query, standalone_server):
    for test in SKIP_LIST:
        if test in sql_query:
            pytest.skip("Test matches skip-list: " + test)
            return

    tcp_port = standalone_server.tcp_port

    query_path = sql_query + ".sql"
    reference_path = sql_query + ".reference"

    if not os.path.exists(reference_path):
        pytest.skip('No .reference file found')

    with open(query_path, 'r') as file:
        query = file.read()
    with open(reference_path, 'rb') as file:
        reference = file.read()

    random_name = 'test_{random}'.format(random=random_str())
    run_client(bin_prefix, tcp_port, 'default', 'CREATE DATABASE {random};'.format(random=random_name), b'')

    run_client(bin_prefix, tcp_port, random_name, query, reference, {random_name: 'default'})

    query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);"
    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n')

    query = 'DROP DATABASE {random};'.format(random=random_name)
    run_client(bin_prefix, tcp_port, 'default', query, b'')

    query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;"
    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')


def test_shell_query(bin_prefix, shell_query, standalone_server):
    for test in SKIP_LIST:
        if test in shell_query:
            pytest.skip("Test matches skip-list: " + test)
            return

    tcp_port = standalone_server.tcp_port

    shell_path = shell_query + ".sh"
    reference_path = shell_query + ".reference"

    if not os.path.exists(reference_path):
        pytest.skip('No .reference file found')

    with open(reference_path, 'rb') as file:
        reference = file.read()

    random_name = 'test_{random}'.format(random=random_str())
    query = 'CREATE DATABASE {random};'.format(random=random_name)
    run_client(bin_prefix, tcp_port, 'default', query, b'')

    run_shell(bin_prefix, standalone_server, random_name, shell_path, reference, {random_name: 'default'})

    query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);"
    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n')

    query = 'DROP DATABASE {random};'.format(random=random_name)
    run_client(bin_prefix, tcp_port, 'default', query, b'')

    query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;"
    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')