ClickHouse/tests/integration/test_distributed_respect_user_timeouts/test.py
tavplubix 3246261da8
Fix logging in test_distributed_respect_user_timeouts (#35575)
* Update test.py

* Update test.py

* Update test.py

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-03-29 15:43:42 +03:00

191 lines
5.8 KiB
Python

import itertools
import os.path
import timeit
import pytest
import logging
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__)
NODES = {"node" + str(i): None for i in (1, 2)}
IS_DEBUG = False
CREATE_TABLES_SQL = """
CREATE DATABASE test;
CREATE TABLE base_table(
node String
)
ENGINE = MergeTree
PARTITION BY node
ORDER BY node;
CREATE TABLE distributed_table
ENGINE = Distributed(test_cluster, default, base_table) AS base_table;
"""
INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}')"
SELECTS_SQL = {
"distributed": "SELECT node FROM distributed_table ORDER BY node",
"remote": (
"SELECT node FROM remote('node1,node2', default.base_table) " "ORDER BY node"
),
}
EXCEPTION_NETWORK = "DB::NetException: "
EXCEPTION_TIMEOUT = "Timeout exceeded while reading from socket ("
EXCEPTION_CONNECT = "Timeout: connect timed out: "
TIMEOUT_MEASUREMENT_EPS = 0.01
EXPECTED_BEHAVIOR = {
"default": {
"times": 3,
"timeout": 1,
},
"ready_to_wait": {
"times": 5,
"timeout": 3,
},
}
TIMEOUT_DIFF_UPPER_BOUND = {
"default": {
"distributed": 5.5,
"remote": 2.5,
},
"ready_to_wait": {
"distributed": 3,
"remote": 2.0,
},
}
def _check_exception(exception, expected_tries=3):
lines = exception.split("\n")
assert len(lines) > 4, "Unexpected exception (expected: timeout info)"
assert lines[0].startswith("Received exception from server (version")
assert lines[1].startswith("Code: 279")
assert lines[1].endswith("All connection tries failed. Log: ")
assert lines[2] == "", "Unexpected exception text (expected: empty line)"
for i, line in enumerate(lines[3 : 3 + expected_tries]):
expected_lines = (
"Code: 209. " + EXCEPTION_NETWORK + EXCEPTION_TIMEOUT,
"Code: 209. " + EXCEPTION_NETWORK + EXCEPTION_CONNECT,
EXCEPTION_TIMEOUT,
)
assert any(
line.startswith(expected) for expected in expected_lines
), 'Unexpected exception "{}" at one of the connection attempts'.format(line)
assert lines[3 + expected_tries] == "", "Wrong number of connect attempts"
@pytest.fixture(scope="module", params=["configs", "configs_secure"])
def started_cluster(request):
cluster = ClickHouseCluster(__file__, request.param)
cluster.__with_ssl_config = request.param == "configs_secure"
main_configs = []
main_configs += [os.path.join(request.param, "config.d/remote_servers.xml")]
if cluster.__with_ssl_config:
main_configs += [os.path.join(request.param, "server.crt")]
main_configs += [os.path.join(request.param, "server.key")]
main_configs += [os.path.join(request.param, "dhparam.pem")]
main_configs += [os.path.join(request.param, "config.d/ssl_conf.xml")]
user_configs = [os.path.join(request.param, "users.d/set_distributed_defaults.xml")]
for name in NODES:
NODES[name] = cluster.add_instance(
name, main_configs=main_configs, user_configs=user_configs
)
try:
cluster.start()
if cluster.instances["node1"].is_debug_build():
global IS_DEBUG
IS_DEBUG = True
logging.warning(
"Debug build is too slow to show difference in timings. We disable checks."
)
for node_id, node in list(NODES.items()):
node.query(CREATE_TABLES_SQL)
node.query(INSERT_SQL_TEMPLATE.format(node_id=node_id))
yield cluster
finally:
cluster.shutdown()
def _check_timeout_and_exception(node, user, query_base, query):
repeats = EXPECTED_BEHAVIOR[user]["times"]
extra_repeats = 1
# Table function remote() are executed two times.
# It tries to get table structure from remote shards.
# On 'node2' it will firstly try to get structure from 'node1' (which is not available),
# so there are 1 extra connection attempts for 'node2' and 'remote'
if node.name == "node2" and query_base == "remote":
extra_repeats = 2
expected_timeout = EXPECTED_BEHAVIOR[user]["timeout"] * repeats * extra_repeats
start = timeit.default_timer()
exception = node.query_and_get_error(query, user=user)
# And it should timeout no faster than:
measured_timeout = timeit.default_timer() - start
if not IS_DEBUG:
assert expected_timeout - measured_timeout <= TIMEOUT_MEASUREMENT_EPS
assert (
measured_timeout - expected_timeout
<= TIMEOUT_DIFF_UPPER_BOUND[user][query_base]
)
# And exception should reflect connection attempts:
_check_exception(exception, repeats)
@pytest.mark.parametrize(
("first_user", "node_name", "query_base"),
tuple(itertools.product(EXPECTED_BEHAVIOR, NODES, SELECTS_SQL)),
)
def test_reconnect(started_cluster, node_name, first_user, query_base):
node = NODES[node_name]
query = SELECTS_SQL[query_base]
if started_cluster.__with_ssl_config:
query = query.replace("remote(", "remoteSecure(")
# Everything is up, select should work:
assert TSV(node.query(query, user=first_user)) == TSV("node1\nnode2")
with PartitionManager() as pm:
# Break the connection.
pm.partition_instances(*list(NODES.values()))
# Now it shouldn't:
_check_timeout_and_exception(node, first_user, query_base, query)
# Other user should have different timeout and exception
_check_timeout_and_exception(
node,
"default" if first_user != "default" else "ready_to_wait",
query_base,
query,
)
# select should work again:
assert TSV(node.query(query, user=first_user)) == TSV("node1\nnode2")