thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

add tests for mutations cleaner [#CLICKHOUSE-3828]

Browse Source
This commit is contained in:
Alexey Zatelepin 2018-08-01 22:01:58 +03:00 committed by alexey-milovidov
parent 7062807d8c
commit a70301d1f2
5 changed files with 101 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
dbms/tests/queries/0_stateless/00652_mergetree_mutations.lib Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
function wait_for_mutation()
{
local table=$1
local mutation_id=$2
for i in {1..100}
do
sleep 0.1
if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT is_done FROM system.mutations WHERE table='$table' AND mutation_id='$mutation_id'") -eq 1 ]]; then
break
fi
if [[ $i -eq 100 ]]; then
echo "Timed out while waiting for mutation to execute!"
fi
done
}

3
dbms/tests/queries/0_stateless/00652_mergetree_mutations.reference
View File

@ -8,3 +8,6 @@ Query involving aliases should fail on submission
mutation_1.txt DELETE WHERE x = 1 [''] [1] 0 1
mutation_5.txt DELETE WHERE (x % 2) = 1 [''] [5] 0 1
mutation_6.txt DELETE WHERE s = \'d\' [''] [6] 0 1
*** Test mutations cleaner ***
mutation_3.txt DELETE WHERE x = 2 1
mutation_4.txt DELETE WHERE x = 3 1

43
dbms/tests/queries/0_stateless/00652_mergetree_mutations.sh
View File

@ -3,6 +3,8 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
. $CURDIR/00652_mergetree_mutations.lib
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations"
${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations(d Date, x UInt32, s String, a UInt32 ALIAS x + 1) ENGINE MergeTree(d, intDiv(x, 10), 8192)"
@ -31,18 +33,8 @@ ${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations DELETE WHERE s = 'd'"
${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations(d, x, s) VALUES \
('2000-01-01', 5, 'e'), ('2000-02-01', 5, 'e')"
# Wait until all mutations are done.
for i in {1..100}
do
sleep 0.1
if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT sum(is_done) FROM system.mutations WHERE table='mutations'") -eq 3 ]]; then
break
fi
if [[ $i -eq 100 ]]; then
echo "Timed out while waiting for mutations to execute!"
fi
done
# Wait until the last mutation is done.
wait_for_mutation "mutations" "mutation_6.txt"
# Check that the table contains only the data that should not be deleted.
${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations ORDER BY d, x"
@ -50,4 +42,31 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations ORDER BY d, x"
${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \
FROM system.mutations WHERE table = 'mutations' ORDER BY mutation_id"
${CLICKHOUSE_CLIENT} --query="SELECT '*** Test mutations cleaner ***'"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner"
# Create a table with finished_mutations_to_keep = 2
${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner(x UInt32) ENGINE MergeTree ORDER BY x SETTINGS finished_mutations_to_keep = 2"
# Insert some data
${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner(x) VALUES (1), (2), (3), (4)"
# Add some mutations and wait for their execution
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 1"
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 2"
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner DELETE WHERE x = 3"
wait_for_mutation "mutations_cleaner" "mutation_4.txt"
# Sleep and then do an INSERT to wakeup the background task that will clean up the old mutations
sleep 1
${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner(x) VALUES (4)"
sleep 0.1
# Check that the first mutation is cleaned
${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, is_done FROM system.mutations WHERE table = 'mutations_cleaner' ORDER BY mutation_id"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner"

4
dbms/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.reference
View File

@ -7,3 +7,7 @@ Query should fail 2
0000000000 DELETE WHERE x = 1 [] [] 0 1
0000000001 DELETE WHERE (x % 2) = 1 ['200001','200002'] [2,1] 0 1
0000000002 DELETE WHERE s = \'d\' ['200001','200002'] [3,2] 0 1
*** Test mutations cleaner ***
0000000001 DELETE WHERE x = 2 1
0000000002 DELETE WHERE x = 3 1
0000000003 DELETE WHERE x = 4 0

55
dbms/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh
View File

@ -3,6 +3,8 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
. $CURDIR/00652_mergetree_mutations.lib
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_r1"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_r2"
@ -31,18 +33,8 @@ ${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_r1 DELETE WHERE s = 'd'
${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_r1(d, x, s) VALUES \
('2000-01-01', 5, 'e'), ('2000-02-01', 5, 'e')"
# Wait until all mutations are done.
for i in {1..100}
do
sleep 0.1
if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT sum(is_done) FROM system.mutations WHERE table='mutations_r2'") -eq 3 ]]; then
break
fi
if [[ $i -eq 100 ]]; then
echo "Timed out while waiting for mutations to execute!"
fi
done
# Wait until the last mutation is done.
wait_for_mutation "mutations_r2" "0000000002"
# Check that the table contains only the data that should not be deleted.
${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations_r2 ORDER BY d, x"
@ -50,5 +42,44 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM test.mutations_r2 ORDER BY d, x"
${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \
FROM system.mutations WHERE table = 'mutations_r2' ORDER BY mutation_id"
${CLICKHOUSE_CLIENT} --query="SELECT '*** Test mutations cleaner ***'"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner_r1"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mutations_cleaner_r2"
# Create 2 replicas with finished_mutations_to_keep = 2
${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/mutations_cleaner', 'r1') ORDER BY x SETTINGS \
finished_mutations_to_keep = 2,
cleanup_delay_period = 1,
cleanup_delay_period_random_add = 0"
${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/mutations_cleaner', 'r2') ORDER BY x SETTINGS \
finished_mutations_to_keep = 2,
cleanup_delay_period = 1,
cleanup_delay_period_random_add = 0"
# Insert some data
${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mutations_cleaner_r1(x) VALUES (1), (2), (3), (4)"
# Add some mutations and wait for their execution
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 1"
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 2"
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 3"
wait_for_mutation "mutations_cleaner_r2" "0000000002"
# Add another mutation and prevent its execution on the second replica
${CLICKHOUSE_CLIENT} --query="SYSTEM STOP REPLICATION QUEUES test.mutations_cleaner_r2"
${CLICKHOUSE_CLIENT} --query="ALTER TABLE test.mutations_cleaner_r1 DELETE WHERE x = 4"
# Sleep for more than cleanup_delay_period
sleep 1.5
# Check that the first mutation is cleaned
${CLICKHOUSE_CLIENT} --query="SELECT mutation_id, command, is_done FROM system.mutations WHERE table = 'mutations_cleaner_r2' ORDER BY mutation_id"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_r1"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_r2"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner_r1"
${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mutations_cleaner_r2"