ClickHouse/tests/integration/test_distributed_respect_user_timeouts/test.py

import itertools
import os.path
import timeit

import pytest
import logging
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV

cluster = ClickHouseCluster(__file__)

NODES = {'node' + str(i): None for i in (1, 2)}

IS_DEBUG = False

CREATE_TABLES_SQL = '''
CREATE DATABASE test;

CREATE TABLE base_table(
    node String
)
ENGINE = MergeTree
PARTITION BY node
ORDER BY node;

CREATE TABLE distributed_table
ENGINE = Distributed(test_cluster, default, base_table) AS base_table;
'''

INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}')"

SELECTS_SQL = {
    'distributed': 'SELECT node FROM distributed_table ORDER BY node',
    'remote': ("SELECT node FROM remote('node1,node2', default.base_table) "
               "ORDER BY node"),
}

EXCEPTION_NETWORK = 'DB::NetException: '
EXCEPTION_TIMEOUT = 'Timeout exceeded while reading from socket ('
EXCEPTION_CONNECT = 'Timeout: connect timed out: '

TIMEOUT_MEASUREMENT_EPS = 0.01

EXPECTED_BEHAVIOR = {
    'default': {
        'times': 3,
        'timeout': 1,
    },
    'ready_to_wait': {
        'times': 5,
        'timeout': 3,
    },
}

TIMEOUT_DIFF_UPPER_BOUND = {
    'default': {
        'distributed': 5.5,
        'remote': 2.5,
    },
    'ready_to_wait': {
        'distributed': 3,
        'remote': 2.0,
    },
}


def _check_exception(exception, expected_tries=3):
    lines = exception.split('\n')

    assert len(lines) > 4, "Unexpected exception (expected: timeout info)"

    assert lines[0].startswith('Received exception from server (version')

    assert lines[1].startswith('Code: 279')
    assert lines[1].endswith('All connection tries failed. Log: ')

    assert lines[2] == '', "Unexpected exception text (expected: empty line)"

    for i, line in enumerate(lines[3:3 + expected_tries]):
        expected_lines = (
            'Code: 209. ' + EXCEPTION_NETWORK + EXCEPTION_TIMEOUT,
            'Code: 209. ' + EXCEPTION_NETWORK + EXCEPTION_CONNECT,
            EXCEPTION_TIMEOUT,
        )

        assert any(line.startswith(expected) for expected in expected_lines), \
            'Unexpected exception "{}" at one of the connection attempts'.format(line)

    assert lines[3 + expected_tries] == '', 'Wrong number of connect attempts'


@pytest.fixture(scope="module", params=["configs", "configs_secure"])
def started_cluster(request):
    cluster = ClickHouseCluster(__file__)
    cluster.__with_ssl_config = request.param == "configs_secure"
    main_configs = []
    main_configs += [os.path.join(request.param, "config.d/remote_servers.xml")]
    if cluster.__with_ssl_config:
        main_configs += [os.path.join(request.param, "server.crt")]
        main_configs += [os.path.join(request.param, "server.key")]
        main_configs += [os.path.join(request.param, "dhparam.pem")]
        main_configs += [os.path.join(request.param, "config.d/ssl_conf.xml")]
    user_configs = [os.path.join(request.param, "users.d/set_distributed_defaults.xml")]
    for name in NODES:
        NODES[name] = cluster.add_instance(name, main_configs=main_configs, user_configs=user_configs)
    try:
        cluster.start()

        if cluster.instances["node1"].is_debug_build():
            global IS_DEBUG
            IS_DEBUG = True
            logging.warning("Debug build is too slow to show difference in timings. We disable checks.")

        for node_id, node in list(NODES.items()):
            node.query(CREATE_TABLES_SQL)
            node.query(INSERT_SQL_TEMPLATE.format(node_id=node_id))

        yield cluster

    finally:
        cluster.shutdown()


def _check_timeout_and_exception(node, user, query_base, query):
    repeats = EXPECTED_BEHAVIOR[user]['times']

    extra_repeats = 1
    # Table function remote() are executed two times.
    # It tries to get table structure from remote shards.
    # On 'node2' it will firstly try to get structure from 'node1' (which is not available),
    # so there are 1 extra connection attempts for 'node2' and 'remote'
    if node.name == 'node2' and query_base == 'remote':
        extra_repeats = 2

    expected_timeout = EXPECTED_BEHAVIOR[user]['timeout'] * repeats * extra_repeats

    start = timeit.default_timer()
    exception = node.query_and_get_error(query, user=user)

    # And it should timeout no faster than:
    measured_timeout = timeit.default_timer() - start

    if not IS_DEBUG:
        assert expected_timeout - measured_timeout <= TIMEOUT_MEASUREMENT_EPS
        assert measured_timeout - expected_timeout <= TIMEOUT_DIFF_UPPER_BOUND[user][query_base]

    # And exception should reflect connection attempts:
    _check_exception(exception, repeats)


@pytest.mark.parametrize(
    ('first_user', 'node_name', 'query_base'),
    tuple(itertools.product(EXPECTED_BEHAVIOR, NODES, SELECTS_SQL)),
)
def test_reconnect(started_cluster, node_name, first_user, query_base):
    node = NODES[node_name]
    query = SELECTS_SQL[query_base]
    if started_cluster.__with_ssl_config:
        query = query.replace('remote(', 'remoteSecure(')

    # Everything is up, select should work:
    assert TSV(node.query(query,
                          user=first_user)) == TSV('node1\nnode2')

    with PartitionManager() as pm:
        # Break the connection.
        pm.partition_instances(*list(NODES.values()))

        # Now it shouldn't:
        _check_timeout_and_exception(node, first_user, query_base, query)

        # Other user should have different timeout and exception
        _check_timeout_and_exception(
            node,
            'default' if first_user != 'default' else 'ready_to_wait',
            query_base,
            query,
        )

    # select should work again:
    assert TSV(node.query(query,
                          user=first_user)) == TSV('node1\nnode2')