ClickHouse/tests/integration/test_mutations_hardlinks/test.py

142 lines
5.5 KiB
Python
Raw Normal View History

2020-07-26 15:58:03 +00:00
import os
import time
from multiprocessing.dummy import Pool
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
2020-07-26 15:58:03 +00:00
cluster = ClickHouseCluster(__file__)
2020-09-14 12:49:04 +00:00
node1 = cluster.add_instance('node1', main_configs=['configs/wide_parts_only.xml'])
2020-07-26 15:58:03 +00:00
2020-07-26 15:58:03 +00:00
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def check_hardlinks(table, part_path, column_file, count):
column_path = os.path.join("/var/lib/clickhouse/data/default", table, part_path, column_file)
script = """
export INODE=`ls -i {column_path} | awk '{{print $1}}'`
export COUNT=`find /var/lib/clickhouse -inum $INODE | wc -l`
test $COUNT = {count}
""".format(column_path=column_path, count=count)
node1.exec_in_container(["bash", "-c", script])
def check_exists(table, part_path, column_file):
column_path = os.path.join("/var/lib/clickhouse/data/default", table, part_path, column_file)
node1.exec_in_container(["bash", "-c", "test -f {}".format(column_path)])
def test_update_mutation(started_cluster):
node1.query(
"CREATE TABLE table_for_update(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()")
2020-07-26 15:58:03 +00:00
node1.query("INSERT INTO table_for_update SELECT number, number, toString(number) from numbers(100)")
assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(range(100))
node1.query("ALTER TABLE table_for_update UPDATE value1 = value1 * value1 WHERE 1",
settings={"mutations_sync": "2"})
2020-07-26 15:58:03 +00:00
assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100))
check_hardlinks("table_for_update", "all_1_1_0_2", "key.bin", 2)
check_hardlinks("table_for_update", "all_1_1_0_2", "value2.bin", 2)
check_hardlinks("table_for_update", "all_1_1_0_2", "value1.bin", 1)
node1.query("ALTER TABLE table_for_update UPDATE key=key, value1=value1, value2=value2 WHERE 1",
settings={"mutations_sync": "2"})
2020-07-26 15:58:03 +00:00
assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100))
check_hardlinks("table_for_update", "all_1_1_0_3", "key.bin", 1)
check_hardlinks("table_for_update", "all_1_1_0_3", "value1.bin", 1)
check_hardlinks("table_for_update", "all_1_1_0_3", "value2.bin", 1)
def test_modify_mutation(started_cluster):
node1.query(
"CREATE TABLE table_for_modify(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()")
2020-07-26 15:58:03 +00:00
node1.query("INSERT INTO table_for_modify SELECT number, number, toString(number) from numbers(100)")
assert int(node1.query("SELECT sum(value1) FROM table_for_modify").strip()) == sum(range(100))
node1.query("ALTER TABLE table_for_modify MODIFY COLUMN value2 UInt64", settings={"mutations_sync": "2"})
2020-07-26 15:58:03 +00:00
assert int(node1.query("SELECT sum(value2) FROM table_for_modify").strip()) == sum(range(100))
check_hardlinks("table_for_modify", "all_1_1_0_2", "key.bin", 2)
check_hardlinks("table_for_modify", "all_1_1_0_2", "value1.bin", 2)
check_hardlinks("table_for_modify", "all_1_1_0_2", "value2.bin", 1)
def test_drop_mutation(started_cluster):
node1.query(
"CREATE TABLE table_for_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()")
2020-07-26 15:58:03 +00:00
node1.query("INSERT INTO table_for_drop SELECT number, number, toString(number) from numbers(100)")
assert int(node1.query("SELECT sum(value1) FROM table_for_drop").strip()) == sum(range(100))
node1.query("ALTER TABLE table_for_drop DROP COLUMN value2", settings={"mutations_sync": "2"})
check_hardlinks("table_for_drop", "all_1_1_0_2", "key.bin", 2)
check_hardlinks("table_for_drop", "all_1_1_0_2", "value1.bin", 2)
with pytest.raises(Exception):
check_exists("table_for_drop", "all_1_1_0_2", "value2.bin")
with pytest.raises(Exception):
check_exists("table_for_drop", "all_1_1_0_2", "value2.mrk")
def test_delete_and_drop_mutation(started_cluster):
node1.query(
"CREATE TABLE table_for_delete_and_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()")
2020-07-26 15:58:03 +00:00
node1.query("INSERT INTO table_for_delete_and_drop SELECT number, number, toString(number) from numbers(100)")
assert int(node1.query("SELECT sum(value1) FROM table_for_delete_and_drop").strip()) == sum(range(100))
node1.query("SYSTEM STOP MERGES")
def mutate():
node1.query("ALTER TABLE table_for_delete_and_drop DELETE WHERE key % 2 == 0, DROP COLUMN value2")
p = Pool(2)
p.apply_async(mutate)
for _ in range(1, 100):
result = node1.query(
"SELECT COUNT() FROM system.mutations WHERE table = 'table_for_delete_and_drop' and is_done=0")
2020-07-26 15:58:03 +00:00
try:
if int(result.strip()) == 2:
break
except:
2020-10-02 16:54:07 +00:00
print("Result", result)
2020-07-26 15:58:03 +00:00
pass
time.sleep(0.5)
node1.query("SYSTEM START MERGES")
assert_eq_with_retry(node1, "SELECT COUNT() FROM table_for_delete_and_drop",
str(sum(1 for i in range(100) if i % 2 != 0)))
2020-07-26 15:58:03 +00:00
check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "key.bin", 1)
check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "value1.bin", 1)
with pytest.raises(Exception):
check_exists("table_for_delete_and_drop", "all_1_1_0_3", "value2.bin")
with pytest.raises(Exception):
check_exists("table_for_delete_and_drop", "all_1_1_0_3", "value2.mrk")