import logging import random import threading import time from collections import Counter import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', macros={'cluster': 'test1'}, with_zookeeper=True) # Check, that limits on max part size for merges doesn`t affect mutations node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], with_zookeeper=True) node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) node5 = cluster.add_instance('node5', macros={'cluster': 'test3'}, main_configs=["configs/merge_tree_max_parts.xml"]) all_nodes = [node1, node2, node3, node4, node5] def prepare_cluster(): for node in all_nodes: node.query("DROP TABLE IF EXISTS test_mutations SYNC") for node in [node1, node2, node3, node4]: node.query(""" CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d) SETTINGS number_of_free_entries_in_pool_to_execute_mutation=0 """) node5.query( "CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() yield cluster finally: cluster.shutdown() class Runner: def __init__(self, nodes): self.nodes = nodes self.mtx = threading.Lock() self.total_inserted_xs = 0 self.total_inserted_rows = 0 self.total_mutations = 0 self.total_deleted_xs = 0 self.total_deleted_rows = 0 self.current_xs = Counter() self.currently_inserting_xs = Counter() self.currently_deleting_xs = set() self.stop_ev = threading.Event() self.exceptions = [] def do_insert(self, thread_num, partitions_num): self.stop_ev.wait(random.random()) # Each thread inserts a small random number of rows with random year, month 01 and day determined # by the thread number. The idea is to avoid spurious duplicates and to insert into a # nontrivial number of partitions. month = '01' day = str(thread_num + 1).zfill(2) i = 1 while not self.stop_ev.is_set(): xs = [random.randint(1, 10) for _ in range(random.randint(1, 10))] with self.mtx: xs = [x for x in xs if x not in self.currently_deleting_xs] if len(xs) == 0: continue for x in xs: self.currently_inserting_xs[x] += 1 year = 2000 + random.randint(0, partitions_num) date_str = '{year}-{month}-{day}'.format(year=year, month=month, day=day) payload = '' for x in xs: payload += '{date_str} {x} {i}\n'.format(date_str=date_str, x=x, i=i) i += 1 try: logging.debug(f"thread {thread_num}: insert for {date_str}: {xs}") random.choice(self.nodes).query("INSERT INTO test_mutations FORMAT TSV", payload) with self.mtx: for x in xs: self.current_xs[x] += 1 self.total_inserted_xs += sum(xs) self.total_inserted_rows += len(xs) except Exception as e: logging.debug(f"Exception while inserting: {e}") self.exceptions.append(e) finally: with self.mtx: for x in xs: self.currently_inserting_xs[x] -= 1 self.stop_ev.wait(0.2 + random.random() / 5) def do_delete(self, thread_num): self.stop_ev.wait(1.0 + random.random()) while not self.stop_ev.is_set(): chosen = False with self.mtx: if self.current_xs: x = random.choice(list(self.current_xs.elements())) if self.currently_inserting_xs[x] == 0 and x not in self.currently_deleting_xs: chosen = True self.currently_deleting_xs.add(x) to_delete_count = self.current_xs[x] if not chosen: self.stop_ev.wait(0.1 * random.random()) continue try: logging.debug(f"thread {thread_num}: delete {to_delete_count} * {x}") random.choice(self.nodes).query("ALTER TABLE test_mutations DELETE WHERE x = {}".format(x)) with self.mtx: self.total_mutations += 1 self.current_xs[x] -= to_delete_count self.total_deleted_xs += to_delete_count * x self.total_deleted_rows += to_delete_count except Exception as e: logging.debug(f"Exception while deleting: {e}") finally: with self.mtx: self.currently_deleting_xs.remove(x) self.stop_ev.wait(1.0 + random.random() * 2) def wait_for_mutations(nodes, number_of_mutations): for i in range(100): # wait for replication 80 seconds max time.sleep(0.8) def get_done_mutations(node): return int(node.query("SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations'").rstrip()) if all([get_done_mutations(n) == number_of_mutations for n in nodes]): return True return False def test_mutations(started_cluster): prepare_cluster() DURATION_SECONDS = 30 nodes = [node1, node2] runner = Runner(nodes) threads = [] for thread_num in range(5): threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 10))) for thread_num in (11, 12, 13): threads.append(threading.Thread(target=runner.do_delete, args=(thread_num,))) for t in threads: t.start() time.sleep(DURATION_SECONDS) runner.stop_ev.set() for t in threads: t.join() # Sanity check: at least something was inserted and something was deleted assert runner.total_inserted_rows > 0 assert runner.total_mutations > 0 all_done = wait_for_mutations(nodes, runner.total_mutations) logging.debug(f"Total mutations: {runner.total_mutations}") for node in nodes: logging.debug(node.query( "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) assert all_done expected_sum = runner.total_inserted_xs - runner.total_deleted_xs actual_sums = [] for i, node in enumerate(nodes): actual_sums.append(int(node.query("SELECT sum(x) FROM test_mutations").rstrip())) assert actual_sums[i] == expected_sum @pytest.mark.parametrize( ('nodes',), [ ([node5, ],), # MergeTree ([node3, node4],), # ReplicatedMergeTree ] ) def test_mutations_dont_prevent_merges(started_cluster, nodes): prepare_cluster() for year in range(2000, 2016): rows = '' date_str = '{}-01-{}'.format(year, random.randint(1, 10)) for i in range(10): rows += '{} {} {}\n'.format(date_str, random.randint(1, 10), i) nodes[0].query("INSERT INTO test_mutations FORMAT TSV", rows) # will run mutations of 16 parts in parallel, mutations will sleep for about 20 seconds nodes[0].query("ALTER TABLE test_mutations UPDATE i = sleepEachRow(2) WHERE 1") runner = Runner(nodes) threads = [] for thread_num in range(2): threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 0))) # will insert approx 8-10 new parts per 1 second into one partition for t in threads: t.start() all_done = wait_for_mutations(nodes, 1) runner.stop_ev.set() for t in threads: t.join() for node in nodes: logging.debug(node.query( "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) logging.debug(node.query( "SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames")) assert all_done, "All done" assert all([str(e).find("Too many parts") < 0 for e in runner.exceptions])