Update Pytest check (#18972)

* [WIP] * Update skip-list * Update ci_config.json * Do not sync inserts for test * Fix more tests * Fix another test * Enable one more test * More fixed tests * More test fixes * Do not absolutize server path for now * More test fixes * Unset CLICKHOUSE_LOG_COMMENT where necessary * Remove debugging set -e * Fix more tests * Fix test reference * Fix style check
2024-11-23 08:02:02 +00:00 · 2021-02-10 21:09:13 +03:00 · 2021-02-10 21:09:13 +03:00 · be831d09f7
commit be831d09f7
parent ece7e87cfb
36 changed files with 259 additions and 70 deletions
--- a/base/daemon/BaseDaemon.cpp
+++ b/base/daemon/BaseDaemon.cpp
@ -986,7 +986,7 @@ void BaseDaemon::setupWatchdog()
        if (errno == ECHILD)
        {
            logger().information("Child process no longer exists.");
-            _exit(status);
+            _exit(WEXITSTATUS(status));
        }

        if (WIFEXITED(status))
@ -1020,7 +1020,7 @@ void BaseDaemon::setupWatchdog()

        /// Automatic restart is not enabled but you can play with it.
 #if 1
-        _exit(status);
+        _exit(WEXITSTATUS(status));
 #else
        logger().information("Will restart.");
        if (argv0)
--- a/docker/test/stateless_pytest/Dockerfile
+++ b/docker/test/stateless_pytest/Dockerfile
@ -5,7 +5,10 @@ RUN apt-get update -y && \
    apt-get install -y --no-install-recommends \
        python3-pip \
        python3-setuptools \
-        python3-wheel
+        python3-wheel \
+        brotli \
+        netcat-openbsd \
+        zstd

 RUN python3 -m pip install \
    wheel \
@ -15,7 +18,10 @@ RUN python3 -m pip install \
    pytest-randomly \
    pytest-rerunfailures \
    pytest-timeout \
-    pytest-xdist
+    pytest-xdist \
+    pandas \
+    numpy \
+    scipy

 CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
--- a/tests/queries/0_stateless/00302_http_compression.sh
+++ b/tests/queries/0_stateless/00302_http_compression.sh
@ -4,6 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh

+if ! command -v gzip &> /dev/null; then echo "gzip not found" 1>&2; exit 1; fi
+if ! command -v brotli &> /dev/null; then echo "brotli not found" 1>&2; exit 1; fi
+if ! command -v xz &> /dev/null; then echo "xz not found" 1>&2; exit 1; fi
+if ! command -v zstd &> /dev/null; then echo "zstd not found" 1>&2; exit 1; fi
+
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1"                                     -d 'SELECT number FROM system.numbers LIMIT 10';
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=0" -H 'Accept-Encoding: gzip'          -d 'SELECT number FROM system.numbers LIMIT 10';
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&enable_http_compression=1" -H 'Accept-Encoding: gzip'          -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d;
--- a/tests/queries/0_stateless/00474_readonly_settings.sh
+++ b/tests/queries/0_stateless/00474_readonly_settings.sh
@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-CLICKHOUSE_LOG_COMMENT=''
+unset CLICKHOUSE_LOG_COMMENT

 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
--- a/tests/queries/0_stateless/00506_union_distributed.sql
+++ b/tests/queries/0_stateless/00506_union_distributed.sql
@ -1,6 +1,7 @@
-
 -- https://github.com/ClickHouse/ClickHouse/issues/1059

+SET insert_distributed_sync = 1;
+
 DROP TABLE IF EXISTS union1;
 DROP TABLE IF EXISTS union2;
 DROP TABLE IF EXISTS union3;
--- a/tests/queries/0_stateless/00717_low_cardinaliry_distributed_group_by.sql
+++ b/tests/queries/0_stateless/00717_low_cardinaliry_distributed_group_by.sql
@ -1,4 +1,6 @@
+set insert_distributed_sync = 1;
 set allow_suspicious_low_cardinality_types = 1;
+
 DROP TABLE IF EXISTS test_low_null_float;
 DROP TABLE IF EXISTS dist_00717;

--- a/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql
+++ b/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql
@ -1,3 +1,5 @@
+SET insert_distributed_sync = 1;
+
 DROP TABLE IF EXISTS low_cardinality;
 DROP TABLE IF EXISTS low_cardinality_all;

--- a/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql
+++ b/tests/queries/0_stateless/00967_insert_into_distributed_different_types.sql
@ -1,3 +1,5 @@
+set insert_distributed_sync=1;
+
 DROP TABLE IF EXISTS dist_00967;
 DROP TABLE IF EXISTS underlying_00967;

@ -5,7 +7,6 @@ DROP TABLE IF EXISTS underlying_00967;
 SET send_logs_level='error';

 CREATE TABLE dist_00967 (key UInt64) Engine=Distributed('test_shard_localhost', currentDatabase(), underlying_00967);
-- fails for TinyLog()/MergeTree()/... but not for Memory()
 CREATE TABLE underlying_00967 (key Nullable(UInt64)) Engine=TinyLog();
 INSERT INTO dist_00967 SELECT toUInt64(number) FROM system.numbers LIMIT 1;

--- a/tests/queries/0_stateless/01023_materialized_view_query_context.sql
+++ b/tests/queries/0_stateless/01023_materialized_view_query_context.sql
@ -1,5 +1,8 @@
 -- Create dictionary, since dictGet*() uses DB::Context in executeImpl()
 -- (To cover scope of the Context in DB::PushingToViewsBlockOutputStream::process)
+
+set insert_distributed_sync=1;
+
 DROP TABLE IF EXISTS mv;
 DROP DATABASE IF EXISTS dict_in_01023;
 CREATE DATABASE dict_in_01023;
--- a/tests/queries/0_stateless/01046_materialized_view_with_join_over_distributed.sql
+++ b/tests/queries/0_stateless/01046_materialized_view_with_join_over_distributed.sql
@ -1,5 +1,7 @@
 -- from https://github.com/ClickHouse/ClickHouse/issues/5142

+set insert_distributed_sync = 1;
+
 DROP TABLE IF EXISTS t;
 DROP TABLE IF EXISTS t_d;
 DROP TABLE IF EXISTS t_v;
--- a/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql
+++ b/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql
@ -1,3 +1,5 @@
+-- set insert_distributed_sync = 1;  -- see https://github.com/ClickHouse/ClickHouse/issues/18971
+
 DROP TABLE IF EXISTS local_01099_a;
 DROP TABLE IF EXISTS local_01099_b;
 DROP TABLE IF EXISTS distributed_01099_a;
--- a/tests/queries/0_stateless/01187_set_profile_as_setting.sh
+++ b/tests/queries/0_stateless/01187_set_profile_as_setting.sh
@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-CLICKHOUSE_LOG_COMMENT=''
+unset CLICKHOUSE_LOG_COMMENT

 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
--- a/tests/queries/0_stateless/01235_live_view_over_distributed.sql
+++ b/tests/queries/0_stateless/01235_live_view_over_distributed.sql
@ -1,3 +1,4 @@
+set insert_distributed_sync = 1;
 SET allow_experimental_live_view = 1;

 DROP TABLE IF EXISTS lv;
--- a/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql
+++ b/tests/queries/0_stateless/01237_live_view_over_distributed_with_subquery_select_table_alias.sql
@ -1,4 +1,5 @@
 SET allow_experimental_live_view = 1;
+SET insert_distributed_sync = 1;

 DROP TABLE IF EXISTS lv;
 DROP TABLE IF EXISTS visits;
--- a/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql
+++ b/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql
@ -1,3 +1,5 @@
+set insert_distributed_sync = 1;
+
 DROP TABLE IF EXISTS visits;
 DROP TABLE IF EXISTS visits_dist;

--- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh
+++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh
@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -ue

-CLICKHOUSE_LOG_COMMENT=''
+unset CLICKHOUSE_LOG_COMMENT

 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
--- a/tests/queries/0_stateless/01475_read_subcolumns.sql
+++ b/tests/queries/0_stateless/01475_read_subcolumns.sql
@ -61,3 +61,8 @@ SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'FileOpen')]
 FROM system.query_log
 WHERE (type = 'QueryFinish') AND (lower(query) LIKE lower('SELECT m.% FROM %t_map%'))
    AND event_time > now() - INTERVAL 10 SECOND AND current_database = currentDatabase();
+
+DROP TABLE t_arr;
+DROP TABLE t_nul;
+DROP TABLE t_tup;
+DROP TABLE t_map;
--- a/tests/queries/0_stateless/01475_read_subcolumns_2.sql
+++ b/tests/queries/0_stateless/01475_read_subcolumns_2.sql
@ -47,3 +47,5 @@ SELECT t.a.size0, t.a.null, t.u, t.s, t.s.null FROM subcolumns;
 SELECT sumArray(arr.null), sum(arr.size0) FROM subcolumns;
 SELECT arr2, arr2.size0, arr2.size1, arr2.null FROM subcolumns;
 -- SELECT nested.col1, nested.col2, nested.size0, nested.size0, nested.col2.null FROM subcolumns;
+
+DROP TABLE subcolumns;
--- a/tests/queries/0_stateless/01475_read_subcolumns_3.sql
+++ b/tests/queries/0_stateless/01475_read_subcolumns_3.sql
@ -37,3 +37,5 @@ SELECT count() FROM map_subcolumns PREWHERE has(m.keys, 'b');

 SELECT id, m.size0 FROM map_subcolumns;
 SELECT count() FROM map_subcolumns WHERE m.size0 > 2;
+
+DROP TABLE map_subcolumns;
--- a/tests/queries/0_stateless/01475_read_subcolumns_storages.sh
+++ b/tests/queries/0_stateless/01475_read_subcolumns_storages.sh
@ -23,3 +23,5 @@ for engine in "${ENGINES[@]}"; do
    $CLICKHOUSE_CLIENT --query "SELECT * FROM subcolumns"
    $CLICKHOUSE_CLIENT --query "SELECT n, n.null, a1, a1.size0, a2, a2.size0, a2.size1, a2.size2, a3, a3.size0, a3.null, t, t.s, t.v, m, m.keys, m.values FROM subcolumns"
 done
+
+$CLICKHOUSE_CLIENT -q "DROP TABLE subcolumns"
--- a/tests/queries/0_stateless/01526_initial_query_id.sh
+++ b/tests/queries/0_stateless/01526_initial_query_id.sh
@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -ue

-CLICKHOUSE_LOG_COMMENT=''
+unset CLICKHOUSE_LOG_COMMENT

 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
--- a/tests/queries/0_stateless/01533_multiple_nested.sql
+++ b/tests/queries/0_stateless/01533_multiple_nested.sql
@ -64,3 +64,5 @@ SELECT id % 10, sum(length(col1)), sumArray(arrayMap(x -> length(x), col1.n.b))

 SELECT arraySum(col1.a), arrayMap(x -> x * x * 2, col1.a) FROM nested ORDER BY id LIMIT 5;
 SELECT untuple(arrayJoin(arrayJoin(col1.n))) FROM nested ORDER BY id LIMIT 10 OFFSET 10;
+
+DROP TABLE nested;
--- a/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh
+++ b/tests/queries/0_stateless/01548_parallel_parsing_max_memory.sh
@ -4,6 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh

-yes http://foobarfoobarfoobarfoobarfoobarfoobarfoobar.com | head -c1G > 1g.csv
+yes http://foobarfoobarfoobarfoobarfoobarfoobarfoobar.com | head -c1G > ${CLICKHOUSE_TMP}/1g.csv

-$CLICKHOUSE_LOCAL --stacktrace --input_format_parallel_parsing=1 --max_memory_usage=100Mi -q "select count() from file('1g.csv', 'TSV', 'URL String')"
+$CLICKHOUSE_LOCAL --stacktrace --input_format_parallel_parsing=1 --max_memory_usage=100Mi -q "select count() from file('${CLICKHOUSE_TMP}/1g.csv', 'TSV', 'URL String')"
--- a/tests/queries/0_stateless/01564_test_hint_woes.reference
+++ b/tests/queries/0_stateless/01564_test_hint_woes.reference
@ -29,3 +29,5 @@ INSERT INTO t0(c0, c1) VALUES ("1",1) ; -- { clientError 47 }
 INSERT INTO t0(c0, c1) VALUES ('1', 1) ;
 -- the return code must be zero after the final query has failed with expected error
 insert into values_01564 values (11); -- { serverError 469 }
+drop table t0;
+drop table values_01564;
--- a/tests/queries/0_stateless/01564_test_hint_woes.sql
+++ b/tests/queries/0_stateless/01564_test_hint_woes.sql
@ -49,3 +49,6 @@ INSERT INTO t0(c0, c1) VALUES ('1', 1) ;

 -- the return code must be zero after the final query has failed with expected error
 insert into values_01564 values (11); -- { serverError 469 }
+
+drop table t0;
+drop table values_01564;
--- a/tests/queries/0_stateless/01586_storage_join_low_cardinality_key.sql
+++ b/tests/queries/0_stateless/01586_storage_join_low_cardinality_key.sql
@ -9,3 +9,5 @@ INSERT INTO low_card VALUES ( '1' );
 SELECT * FROM low_card;
 SELECT * FROM low_card WHERE lc = '1';
 SELECT CAST(lc AS String) FROM low_card;
+
+DROP TABLE low_card;
--- a/tests/queries/0_stateless/01606_git_import.sh
+++ b/tests/queries/0_stateless/01606_git_import.sh
@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

 # Clone some not too large repository and create a database from it.

+cd $CLICKHOUSE_TMP || exit
+
 # Protection for network errors
 for _ in {1..10}; do
    rm -rf ./clickhouse-odbc
--- a/tests/queries/0_stateless/01621_clickhouse_compressor.sh
+++ b/tests/queries/0_stateless/01621_clickhouse_compressor.sh
@ -7,27 +7,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

 set -e

-TEMP_DIR="$(mktemp -d /tmp/clickhouse.test..XXXXXX)"
-cd "${TEMP_DIR:?}"
-
-function cleanup()
-{
-    rm -fr "${TEMP_DIR:?}"
-}
-trap cleanup EXIT
-
 # This is random garbage, so compression ratio will be very low.
-tr -cd 'a-z0-9' < /dev/urandom | head -c1M > input
+tr -cd 'a-z0-9' < /dev/urandom | head -c1M > ${CLICKHOUSE_TMP}/input

 # stdin/stdout streams
-$CLICKHOUSE_COMPRESSOR < input > output
-diff -q <($CLICKHOUSE_COMPRESSOR --decompress < output) input
+$CLICKHOUSE_COMPRESSOR < ${CLICKHOUSE_TMP}/input > ${CLICKHOUSE_TMP}/output
+diff -q <($CLICKHOUSE_COMPRESSOR --decompress < ${CLICKHOUSE_TMP}/output) ${CLICKHOUSE_TMP}/input

 # positional arguments, and that fact that input/output will be overwritten
-$CLICKHOUSE_COMPRESSOR input output
-diff -q <($CLICKHOUSE_COMPRESSOR --decompress output) input
+$CLICKHOUSE_COMPRESSOR ${CLICKHOUSE_TMP}/input ${CLICKHOUSE_TMP}/output
+diff -q <($CLICKHOUSE_COMPRESSOR --decompress ${CLICKHOUSE_TMP}/output) ${CLICKHOUSE_TMP}/input

 # --offset-in-decompressed-block
-diff -q <($CLICKHOUSE_COMPRESSOR --decompress --offset-in-decompressed-block 10 output) <(tail -c+$((10+1)) input)
+diff -q <($CLICKHOUSE_COMPRESSOR --decompress --offset-in-decompressed-block 10 ${CLICKHOUSE_TMP}/output) <(tail -c+$((10+1)) ${CLICKHOUSE_TMP}/input)

 # TODO: --offset-in-compressed-file using some .bin file (via clickhouse-local + check-marks)
--- a/tests/queries/0_stateless/01622_defaults_for_url_engine.sh
+++ b/tests/queries/0_stateless/01622_defaults_for_url_engine.sh
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

 PORT="$(($RANDOM%63000+2001))"

-TEMP_FILE="$CURDIR/01622_defaults_for_url_engine.tmp"
+TEMP_FILE="${CLICKHOUSE_TMP}/01622_defaults_for_url_engine.tmp"

 function thread1
 {
--- a/tests/queries/0_stateless/01652_ignore_and_low_cardinality.sql
+++ b/tests/queries/0_stateless/01652_ignore_and_low_cardinality.sql
@ -4,3 +4,5 @@ SELECT ignore(10, ignore(*), ignore(ignore(-2, 1025, *)), NULL, *), * FROM lc_nu


 SELECT ignore(toLowCardinality(1), toLowCardinality(2), 3);
+
+DROP TABLE lc_null_int8_defnull;
--- a/tests/queries/0_stateless/01656_test_query_log_factories_info.sql
+++ b/tests/queries/0_stateless/01656_test_query_log_factories_info.sql
@ -56,3 +56,5 @@ WHERE current_database = currentDatabase() AND type == 'QueryFinish' AND (query
 ORDER BY query_start_time DESC LIMIT 1 FORMAT TabSeparatedWithNames;
 SELECT '';

+DROP TABLE test_query_log_factories_info1.memory_table;
+DROP DATABASE test_query_log_factories_info1;
--- a/tests/queries/0_stateless/01665_merge_tree_min_for_concurrent_read.sql
+++ b/tests/queries/0_stateless/01665_merge_tree_min_for_concurrent_read.sql
@ -4,3 +4,5 @@ INSERT INTO data_01655 VALUES (1);
 SELECT * FROM data_01655 SETTINGS merge_tree_min_rows_for_concurrent_read=0, merge_tree_min_bytes_for_concurrent_read=0;
 -- UINT64_MAX
 SELECT * FROM data_01655 SETTINGS merge_tree_min_rows_for_concurrent_read=18446744073709551615, merge_tree_min_bytes_for_concurrent_read=18446744073709551615;
+
+DROP TABLE data_01655;
--- a/tests/queries/conftest.py
+++ b/tests/queries/conftest.py
@ -25,6 +25,9 @@ def bin_prefix(cmdopts):
    prefix = 'clickhouse'
    if cmdopts['builddir'] is not None:
        prefix = os.path.join(cmdopts['builddir'], 'programs', prefix)
+    # FIXME: does this hangs the server start for some reason?
+    # if not os.path.isabs(prefix):
+    #     prefix = os.path.abspath(prefix)
    return prefix


--- a/tests/queries/query_test.py
+++ b/tests/queries/query_test.py
@ -14,13 +14,10 @@ SKIP_LIST = [
    "00987_distributed_stack_overflow",

    # just fail
-    "00302_http_compression",
-    "00463_long_sessions_in_http_interface",
    "00505_secure",
    "00505_shard_secure",
-    "00506_union_distributed",  # flaky
    "00646_url_engine",
-    "00821_distributed_storage_with_join_on.sql",  # flaky
+    "00725_memory_tracking",  # BROKEN
    "00834_cancel_http_readonly_queries_on_client_close",
    "00933_test_fix_extra_seek_on_compressed_cache",
    "00965_logs_level_bugfix",
@ -30,10 +27,6 @@ SKIP_LIST = [
    "01014_lazy_database_concurrent_recreate_reattach_and_show_tables",
    "01018_Distributed__shard_num",
    "01018_ip_dictionary",
-    "01023_materialized_view_query_context",  # flaky
-    "01035_lc_empty_part_bug",  # flaky
-    "01037_polygon_dicts_simple_functions.sh",  # flaky
-    "01046_materialized_view_with_join_over_distributed",  # flaky
    "01050_clickhouse_dict_source_with_subquery",
    "01053_ssd_dictionary",
    "01054_cache_dictionary_overflow_cell",
@ -43,25 +36,25 @@ SKIP_LIST = [
    "01086_odbc_roundtrip",
    "01088_benchmark_query_id",
    "01098_temporary_and_external_tables",
-    "01099_parallel_distributed_insert_select",  # flaky
+    "01099_parallel_distributed_insert_select",
    "01103_check_cpu_instructions_at_startup",
    "01114_database_atomic",
    "01148_zookeeper_path_macros_unfolding",
-    "01193_metadata_loading.sh",  # flaky
-    "01274_alter_rename_column_distributed",  # flaky
+    "01181_db_atomic_drop_on_cluster",  # tcp port in reference
    "01280_ssd_complex_key_dictionary",
    "01293_client_interactive_vertical_multiline",  # expect-test
    "01293_client_interactive_vertical_singleline",  # expect-test
+    "01293_system_distribution_queue",  # FLAKY
    "01293_show_clusters",
    "01294_lazy_database_concurrent_recreate_reattach_and_show_tables",
    "01294_system_distributed_on_cluster",
    "01300_client_save_history_when_terminated",  # expect-test
    "01304_direct_io",
    "01306_benchmark_json",
+    "01035_lc_empty_part_bug",  # FLAKY
    "01320_create_sync_race_condition_zookeeper",
    "01355_CSV_input_format_allow_errors",
    "01370_client_autocomplete_word_break_characters",  # expect-test
-    "01375_storage_file_tsv_csv_with_names_write_prefix",  # flaky
    "01376_GROUP_BY_injective_elimination_dictGet",
    "01393_benchmark_secure_port",
    "01418_custom_settings",
@ -72,6 +65,7 @@ SKIP_LIST = [
    "01507_clickhouse_server_start_with_embedded_config",
    "01514_distributed_cancel_query_on_error",
    "01520_client_print_query_id",  # expect-test
+    "01526_client_start_and_exit",  # expect-test
    "01527_dist_sharding_key_dictGet_reload",
    "01545_url_file_format_settings",
    "01553_datetime64_comparison",
@ -79,15 +73,16 @@ SKIP_LIST = [
    "01558_ttest_scipy",
    "01561_mann_whitney_scipy",
    "01582_distinct_optimization",
-    "01586_storage_join_low_cardinality_key",
-    "01599_multiline_input_and_singleline_comments",
-    "01600_benchmark_query",
+    "01599_multiline_input_and_singleline_comments",  # expect-test
    "01601_custom_tld",
-    "01601_proxy_protocol",
+    "01610_client_spawn_editor",  # expect-test
+    "01676_clickhouse_client_autocomplete",  # expect-test (partially)
+    "01683_text_log_deadlock",  # secure tcp
 ]


 def check_result(result, error, return_code, reference, replace_map):
+    if replace_map:
        for old, new in replace_map.items():
            result = result.replace(old.encode('utf-8'), new.encode('utf-8'))

@ -106,9 +101,9 @@ def check_result(result, error, return_code, reference, replace_map):
            pytrace=False)


-def run_client(bin_prefix, port, query, reference, replace_map={}):
+def run_client(bin_prefix, port, database, query, reference, replace_map=None):
    # We can't use `text=True` since some tests may return binary data
-    client = subprocess.Popen([bin_prefix + '-client', '--port', str(port), '-m', '-n', '--testmode'],
+    client = subprocess.Popen([bin_prefix + '-client', '--port', str(port), '-d', database, '-m', '-n', '--testmode'],
                              stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    result, error = client.communicate(query.encode('utf-8'))
    assert client.returncode is not None, "Client should exit after processing all queries"
@ -116,12 +111,13 @@ def run_client(bin_prefix, port, query, reference, replace_map={}):
    check_result(result, error, client.returncode, reference, replace_map)


-def run_shell(bin_prefix, server, database, path, reference, replace_map={}):
+def run_shell(bin_prefix, server, database, path, reference, replace_map=None):
    env = {
        'CLICKHOUSE_BINARY': bin_prefix,
        'CLICKHOUSE_DATABASE': database,
        'CLICKHOUSE_PORT_TCP': str(server.tcp_port),
        'CLICKHOUSE_PORT_TCP_SECURE': str(server.tcps_port),
+        'CLICKHOUSE_PORT_TCP_WITH_PROXY': str(server.proxy_port),
        'CLICKHOUSE_PORT_HTTP': str(server.http_port),
        'CLICKHOUSE_PORT_INTERSERVER': str(server.inter_port),
        'CLICKHOUSE_TMP': server.tmp_dir,
@ -136,6 +132,7 @@ def run_shell(bin_prefix, server, database, path, reference, replace_map={}):

 def random_str(length=10):
    alphabet = string.ascii_lowercase + string.digits
+    random.seed(os.urandom(8))
    return ''.join(random.choice(alphabet) for _ in range(length))


@ -159,17 +156,18 @@ def test_sql_query(bin_prefix, sql_query, standalone_server):
        reference = file.read()

    random_name = 'test_{random}'.format(random=random_str())
-    query = 'CREATE DATABASE {random}; USE {random}; {query}'.format(random=random_name, query=query)
-    run_client(bin_prefix, tcp_port, query, reference, {random_name: 'default'})
+    run_client(bin_prefix, tcp_port, 'default', 'CREATE DATABASE {random};'.format(random=random_name), b'')
+
+    run_client(bin_prefix, tcp_port, random_name, query, reference, {random_name: 'default'})

    query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);"
-    run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED TABLES\n')
+    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n')

    query = 'DROP DATABASE {random};'.format(random=random_name)
-    run_client(bin_prefix, tcp_port, query, b'')
+    run_client(bin_prefix, tcp_port, 'default', query, b'')

    query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;"
-    run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')
+    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')


 def test_shell_query(bin_prefix, shell_query, standalone_server):
@ -191,15 +189,15 @@ def test_shell_query(bin_prefix, shell_query, standalone_server):

    random_name = 'test_{random}'.format(random=random_str())
    query = 'CREATE DATABASE {random};'.format(random=random_name)
-    run_client(bin_prefix, tcp_port, query, b'')
+    run_client(bin_prefix, tcp_port, 'default', query, b'')

    run_shell(bin_prefix, standalone_server, random_name, shell_path, reference, {random_name: 'default'})

    query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);"
-    run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED TABLES\n')
+    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n')

    query = 'DROP DATABASE {random};'.format(random=random_name)
-    run_client(bin_prefix, tcp_port, query, b'')
+    run_client(bin_prefix, tcp_port, 'default', query, b'')

    query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;"
-    run_client(bin_prefix, tcp_port, query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')
+    run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n')
--- a/tests/queries/server.py
+++ b/tests/queries/server.py
@ -37,6 +37,7 @@ class ServerThread(threading.Thread):
        self.tcps_port = port_base + 4
        self.https_port = port_base + 5
        self.odbc_port = port_base + 6
+        self.proxy_port = port_base + 7

        self._args = [
            '--config-file={config_path}'.format(config_path=self.server_config),
@ -44,6 +45,7 @@ class ServerThread(threading.Thread):
            '--tcp_port={tcp_port}'.format(tcp_port=self.tcp_port),
            '--http_port={http_port}'.format(http_port=self.http_port),
            '--interserver_http_port={inter_port}'.format(inter_port=self.inter_port),
+            '--tcp_with_proxy_port={proxy_port}'.format(proxy_port=self.proxy_port),
            # TODO: SSL certificate is not specified '--tcp_port_secure={tcps_port}'.format(tcps_port=self.tcps_port),
        ]

@ -76,8 +78,8 @@ class ServerThread(threading.Thread):
                    print('Successful server response:', s.recv(1024))  # FIXME: read whole buffered response
                    s.shutdown(socket.SHUT_RDWR)
                    s.close()
-                except Exception as e:
-                    print('Failed to connect to server:', e, file=sys.stderr)
+                except Exception:
+                    # Failed to connect to server - try again
                    continue
                else:
                    break
@ -96,6 +98,10 @@ class ServerThread(threading.Thread):

        self._lock.release()

+        if not retries:
+            print('Failed to start server', file=sys.stderr)
+            return
+
        while self._proc.returncode is None:
            self._proc.communicate()

@ -297,6 +303,10 @@ ServerThread.DEFAULT_SERVER_CONFIG = \
        <implementation>testkeeper</implementation>
    </zookeeper>

+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+
    <part_log>
        <database>system</database>
        <table>part_log</table>
@ -1112,6 +1122,136 @@ ServerThread.DEFAULT_DICTIONARIES_CONFIG = \
            </attribute>
        </structure>
    </dictionary>
+
+    <dictionary>
+        <name>simple_executable_cache_dictionary_no_implicit_key</name>
+        <structure>
+            <id>
+                <name>id</name>
+                <type>UInt64</type>
+            </id>
+
+            <attribute>
+                <name>value</name>
+                <type>String</type>
+                <null_value></null_value>
+            </attribute>
+        </structure>
+        <source>
+            <executable>
+                <command>echo "1\tValue"</command>
+                <format>TabSeparated</format>
+                <implicit_key>false</implicit_key>
+            </executable>
+        </source>
+        <layout>
+            <cache>
+                <size_in_cells>10000</size_in_cells>
+            </cache>
+        </layout>
+        <lifetime>300</lifetime>
+    </dictionary>
+
+    <dictionary>
+        <name>simple_executable_cache_dictionary_implicit_key</name>
+        <structure>
+            <id>
+                <name>id</name>
+                <type>UInt64</type>
+            </id>
+
+            <attribute>
+                <name>value</name>
+                <type>String</type>
+                <null_value></null_value>
+            </attribute>
+        </structure>
+        <source>
+            <executable>
+                <command>echo "Value"</command>
+                <format>TabSeparated</format>
+                <implicit_key>true</implicit_key>
+            </executable>
+        </source>
+        <layout>
+            <cache>
+                <size_in_cells>10000</size_in_cells>
+            </cache>
+        </layout>
+        <lifetime>300</lifetime>
+    </dictionary>
+
+    <dictionary>
+        <name>complex_executable_cache_dictionary_no_implicit_key</name>
+        <structure>
+            <key>
+                <attribute>
+                    <name>id</name>
+                    <type>UInt64</type>
+                    <null_value></null_value>
+                </attribute>
+                <attribute>
+                    <name>id_key</name>
+                    <type>String</type>
+                    <null_value></null_value>
+                </attribute>
+            </key>
+            <attribute>
+                <name>value</name>
+                <type>String</type>
+                <null_value></null_value>
+            </attribute>
+        </structure>
+        <source>
+            <executable>
+                <command>echo "1\tFirstKey\tValue"</command>
+                <format>TabSeparated</format>
+                <implicit_key>false</implicit_key>
+            </executable>
+        </source>
+        <layout>
+            <complex_key_cache>
+                <size_in_cells>10000</size_in_cells>
+            </complex_key_cache>
+        </layout>
+        <lifetime>300</lifetime>
+    </dictionary>
+
+    <dictionary>
+        <name>complex_executable_cache_dictionary_implicit_key</name>
+        <structure>
+            <key>
+                <attribute>
+                    <name>id</name>
+                    <type>UInt64</type>
+                    <null_value></null_value>
+                </attribute>
+                <attribute>
+                    <name>id_key</name>
+                    <type>String</type>
+                    <null_value></null_value>
+                </attribute>
+            </key>
+            <attribute>
+                <name>value</name>
+                <type>String</type>
+                <null_value></null_value>
+            </attribute>
+        </structure>
+        <source>
+            <executable>
+                <command>echo "Value"</command>
+                <format>TabSeparated</format>
+                <implicit_key>true</implicit_key>
+            </executable>
+        </source>
+        <layout>
+            <complex_key_cache>
+                <size_in_cells>10000</size_in_cells>
+            </complex_key_cache>
+        </layout>
+        <lifetime>300</lifetime>
+    </dictionary>
 </yandex>
 """

--- a/tests/queries/shell_config.sh
+++ b/tests/queries/shell_config.sh
@ -8,11 +8,12 @@ export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL
 [ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} "
 [ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} "
 [ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
+[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_BENCHMARK_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
 [ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} "
 [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
-[ -n "$CLICKHOUSE_LOG_COMMENT" ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' "
+[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' "
 [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
-[ -n "$CLICKHOUSE_LOG_COMMENT" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' "
+[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' "

 export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"}
 [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client}
@ -62,7 +63,7 @@ else
  export CLICKHOUSE_URL_PARAMS="database=${CLICKHOUSE_DATABASE}"
 fi
 # Note: missing url encoding of the log comment.
-[ -n "$CLICKHOUSE_LOG_COMMENT" ] && export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&log_comment=${CLICKHOUSE_LOG_COMMENT}"
+[ -v CLICKHOUSE_LOG_COMMENT ] && export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&log_comment=${CLICKHOUSE_LOG_COMMENT}"

 export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"}
 export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"}