ClickHouse/tests/queries/0_stateless/replication.lib

#!/usr/bin/env bash
# shellcheck source=./mergetree_mutations.lib
. "$CURDIR"/mergetree_mutations.lib

function try_sync_replicas()
{
    readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$1%' AND engine like '%Replicated%'")
    for t in "${tables_arr[@]}"
    do
        # The size of log may be big, so increase timeout.
        $CLICKHOUSE_CLIENT --receive_timeout 400 -q "SYSTEM SYNC REPLICA $t" || $CLICKHOUSE_CLIENT -q \
            "select 'sync failed, queue:', * from system.replication_queue where database=currentDatabase() and table='$t'" &
    done
    wait
    echo "Replication did not hang: synced all replicas of $1"
}

function check_replication_consistency()
{
    # Forcefully cancel mutations to avoid waiting for them to finish
    ${CLICKHOUSE_CLIENT} --query="KILL MUTATION WHERE database=currentDatabase() AND table like '$1%'" > /dev/null

    # SYNC REPLICA is not enough if some MUTATE_PARTs are not assigned yet
    wait_for_all_mutations "$1%"

    try_sync_replicas "$1"

    res=$($CLICKHOUSE_CLIENT -q \
    "SELECT
        if((countDistinct(data) as c) == 0, 1, c)
    FROM
    (
        SELECT _table, ($2) AS data
        FROM merge(currentDatabase(), '$1') GROUP BY _table
    )")

    echo "Consistency: $res"
    if [ $res -ne 1 ]; then
        echo "Replicas have diverged:"
        $CLICKHOUSE_CLIENT -q "select 'data', _table, $2, arraySort(groupArrayDistinct(_part)) from merge(currentDatabase(), '$1') group by _table"
        $CLICKHOUSE_CLIENT -q "select 'queue', * from system.replication_queue where database=currentDatabase() and table like '$1%'"
        $CLICKHOUSE_CLIENT -q "select 'mutations', * from system.mutations where database=currentDatabase() and table like '$1%'"
        $CLICKHOUSE_CLIENT -q "select 'parts', * from system.parts where database=currentDatabase() and table like '$1%'"
        echo "Good luck with debugging..."
    fi

}