ClickHouse/tests/integration/test_insert_distributed_async_send/test.py

# pylint: disable=unused-argument
# pylint: disable=redefined-outer-name
# pylint: disable=line-too-long

# NOTES:
# - timeout should not be reduced due to bit flip of the corrupted buffer

import pytest

from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException

cluster = ClickHouseCluster(__file__)

# n1 -- distributed_directory_monitor_batch_inserts=1
n1 = cluster.add_instance(
    "n1",
    main_configs=["configs/remote_servers.xml"],
    user_configs=["configs/users.d/batch.xml"],
)
# n2 -- distributed_directory_monitor_batch_inserts=0
n2 = cluster.add_instance(
    "n2",
    main_configs=["configs/remote_servers.xml"],
    user_configs=["configs/users.d/no_batch.xml"],
)

# n3 -- distributed_directory_monitor_batch_inserts=1/distributed_directory_monitor_split_batch_on_failure=1
n3 = cluster.add_instance(
    "n3",
    main_configs=["configs/remote_servers_split.xml"],
    user_configs=[
        "configs/users.d/batch.xml",
        "configs/users.d/split.xml",
    ],
)
# n4 -- distributed_directory_monitor_batch_inserts=0/distributed_directory_monitor_split_batch_on_failure=1
n4 = cluster.add_instance(
    "n4",
    main_configs=["configs/remote_servers_split.xml"],
    user_configs=[
        "configs/users.d/no_batch.xml",
        "configs/users.d/split.xml",
    ],
)

batch_params = pytest.mark.parametrize(
    "batch",
    [
        (1),
        (0),
    ],
)

batch_and_split_params = pytest.mark.parametrize(
    "batch,split",
    [
        (1, 0),
        (0, 0),
        (1, 1),
        (0, 1),
    ],
)


@pytest.fixture(scope="module", autouse=True)
def start_cluster():
    try:
        cluster.start()
        yield cluster
    finally:
        cluster.shutdown()


def create_tables(remote_cluster_name):
    for _, instance in list(cluster.instances.items()):
        instance.query(
            "CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key"
        )
        instance.query(
            f"""
        CREATE TABLE dist AS data
        Engine=Distributed(
            {remote_cluster_name},
            currentDatabase(),
            data,
            key
        )
        """
        )
        # only via SYSTEM FLUSH DISTRIBUTED
        instance.query("SYSTEM STOP DISTRIBUTED SENDS dist")


def drop_tables():
    for _, instance in list(cluster.instances.items()):
        instance.query("DROP TABLE IF EXISTS data")
        instance.query("DROP TABLE IF EXISTS dist")


# return amount of bytes of the 2.bin for n2 shard
def insert_data(node):
    node.query(
        "INSERT INTO dist SELECT number, randomPrintableASCII(100) FROM numbers(10000)",
        settings={
            # do not do direct INSERT, always via SYSTEM FLUSH DISTRIBUTED
            "prefer_localhost_replica": 0,
        },
    )
    path = get_path_to_dist_batch()
    size = int(node.exec_in_container(["bash", "-c", f"wc -c < {path}"]))
    assert size > 1 << 16
    return size


def get_node(batch, split=None):
    if split:
        if batch:
            return n3
        return n4
    if batch:
        return n1
    return n2


def bootstrap(batch, split=None):
    drop_tables()
    create_tables("insert_distributed_async_send_cluster_two_replicas")
    return insert_data(get_node(batch, split))


def get_path_to_dist_batch(file="2.bin"):
    # There are:
    # - /var/lib/clickhouse/data/default/dist/shard1_replica1/1.bin
    # - /var/lib/clickhouse/data/default/dist/shard1_replica2/2.bin
    #
    # @return the file for the n2 shard
    return f"/var/lib/clickhouse/data/default/dist/shard1_replica2/{file}"


def check_dist_after_corruption(truncate, batch, split=None):
    node = get_node(batch, split)

    if batch:
        # In batch mode errors are ignored
        node.query("SYSTEM FLUSH DISTRIBUTED dist")
    else:
        if truncate:
            with pytest.raises(
                QueryRuntimeException, match="Cannot read all data. Bytes read:"
            ):
                node.query("SYSTEM FLUSH DISTRIBUTED dist")
        else:
            with pytest.raises(
                QueryRuntimeException,
                match="Checksum doesn't match: corrupted data. Reference:",
            ):
                node.query("SYSTEM FLUSH DISTRIBUTED dist")

    # send pending files
    # (since we have two nodes and corrupt file for only one of them)
    node.query("SYSTEM FLUSH DISTRIBUTED dist")

    # but there is broken file
    broken = get_path_to_dist_batch("broken")
    node.exec_in_container(["bash", "-c", f"ls {broken}/2.bin"])

    if split:
        assert int(n3.query("SELECT count() FROM data")) == 10000
        assert int(n4.query("SELECT count() FROM data")) == 0
    else:
        assert int(n1.query("SELECT count() FROM data")) == 10000
        assert int(n2.query("SELECT count() FROM data")) == 0


@batch_params
def test_insert_distributed_async_send_success(batch):
    bootstrap(batch)
    node = get_node(batch)
    node.query("SYSTEM FLUSH DISTRIBUTED dist")
    assert int(n1.query("SELECT count() FROM data")) == 10000
    assert int(n2.query("SELECT count() FROM data")) == 10000


@batch_and_split_params
def test_insert_distributed_async_send_truncated_1(batch, split):
    size = bootstrap(batch, split)
    path = get_path_to_dist_batch()
    node = get_node(batch, split)

    new_size = size - 10
    # we cannot use truncate, due to hardlinks
    node.exec_in_container(
        ["bash", "-c", f"mv {path} /tmp/bin && head -c {new_size} /tmp/bin > {path}"]
    )

    check_dist_after_corruption(True, batch, split)


@batch_params
def test_insert_distributed_async_send_truncated_2(batch):
    bootstrap(batch)
    path = get_path_to_dist_batch()
    node = get_node(batch)

    # we cannot use truncate, due to hardlinks
    node.exec_in_container(
        ["bash", "-c", f"mv {path} /tmp/bin && head -c 10000 /tmp/bin > {path}"]
    )

    check_dist_after_corruption(True, batch)


# The difference from the test_insert_distributed_async_send_corrupted_small
# is that small corruption will be seen only on local node
@batch_params
def test_insert_distributed_async_send_corrupted_big(batch):
    size = bootstrap(batch)
    path = get_path_to_dist_batch()

    node = get_node(batch)

    from_original_size = size - 8192
    zeros_size = 8192
    node.exec_in_container(
        [
            "bash",
            "-c",
            f"mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}",
        ]
    )

    check_dist_after_corruption(False, batch)


@batch_params
def test_insert_distributed_async_send_corrupted_small(batch):
    size = bootstrap(batch)
    path = get_path_to_dist_batch()
    node = get_node(batch)

    from_original_size = size - 60
    zeros_size = 60
    node.exec_in_container(
        [
            "bash",
            "-c",
            f"mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}",
        ]
    )

    check_dist_after_corruption(False, batch)


@batch_params
def test_insert_distributed_async_send_different_header(batch):
    """
    Check INSERT Into Distributed() with different headers in *.bin
    If batching will not distinguish headers underlying table will never receive the data.
    """

    drop_tables()
    create_tables("insert_distributed_async_send_cluster_two_shards")

    node = get_node(batch)
    node.query(
        "INSERT INTO dist VALUES (0, 'f')",
        settings={
            "prefer_localhost_replica": 0,
        },
    )
    node.query("ALTER TABLE dist MODIFY COLUMN value UInt64")
    node.query(
        "INSERT INTO dist VALUES (2, 1)",
        settings={
            "prefer_localhost_replica": 0,
        },
    )

    n1.query(
        "ALTER TABLE data MODIFY COLUMN value UInt64",
        settings={
            "mutations_sync": 1,
        },
    )

    if batch:
        # but only one batch will be sent, and first is with UInt64 column, so
        # one rows inserted, and for string ('f') exception will be throw.
        with pytest.raises(
            QueryRuntimeException,
            match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string",
        ):
            node.query("SYSTEM FLUSH DISTRIBUTED dist")
        assert int(n1.query("SELECT count() FROM data")) == 1
        # but once underlying column String, implicit conversion will do the
        # thing, and insert left batch.
        n1.query(
            """
        DROP TABLE data SYNC;
        CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key;
        """
        )
        node.query("SYSTEM FLUSH DISTRIBUTED dist")
        assert int(n1.query("SELECT count() FROM data")) == 1
    else:
        # first send with String ('f'), so zero rows will be inserted
        with pytest.raises(
            QueryRuntimeException,
            match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string",
        ):
            node.query("SYSTEM FLUSH DISTRIBUTED dist")
        assert int(n1.query("SELECT count() FROM data")) == 0
        # but once underlying column String, implicit conversion will do the
        # thing, and insert 2 rows (mixed UInt64 and String).
        n1.query(
            """
        DROP TABLE data SYNC;
        CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key;
        """
        )
        node.query("SYSTEM FLUSH DISTRIBUTED dist")
        assert int(n1.query("SELECT count() FROM data")) == 2

    assert int(n2.query("SELECT count() FROM data")) == 0