mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 16:50:48 +00:00
Add test for interaction of tables with different granularity and different versions
This commit is contained in:
alesapin
parent
4ae502eaa6
commit
80f27968ac
@ -47,7 +47,11 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
|
||||
|
||||
ttl_table = formattedAST(data.ttl_table_ast);
|
||||
skip_indices = data.getIndices().toString();
|
||||
index_granularity_bytes = data.settings.index_granularity_bytes;
|
||||
if (data.canUseAdaptiveGranularity())
|
||||
index_granularity_bytes = data.settings.index_granularity_bytes;
|
||||
else
|
||||
index_granularity_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const
|
||||
@ -114,7 +118,12 @@ void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in)
|
||||
in >> skip_indices >> "\n";
|
||||
|
||||
if (checkString("granularity bytes: ", in))
|
||||
{
|
||||
in >> index_granularity_bytes >> "\n";
|
||||
index_granularity_bytes_found_in_zk = true;
|
||||
}
|
||||
else
|
||||
index_granularity_bytes = 0;
|
||||
}
|
||||
|
||||
ReplicatedMergeTreeTableMetadata ReplicatedMergeTreeTableMetadata::parse(const String & s)
|
||||
@ -227,7 +236,7 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl
|
||||
ErrorCodes::METADATA_MISMATCH);
|
||||
}
|
||||
|
||||
if (index_granularity_bytes != from_zk.index_granularity_bytes)
|
||||
if (from_zk.index_granularity_bytes_found_in_zk && index_granularity_bytes != from_zk.index_granularity_bytes)
|
||||
throw Exception("Existing table metadata in ZooKeeper differs in index granularity bytes."
|
||||
" Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity_bytes) +
|
||||
", local: " + DB::toString(index_granularity_bytes),
|
||||
|
||||
@ -26,8 +26,8 @@ struct ReplicatedMergeTreeTableMetadata
|
||||
String partition_key;
|
||||
String sorting_key;
|
||||
String skip_indices;
|
||||
UInt64 index_granularity_bytes;
|
||||
String ttl_table;
|
||||
UInt64 index_granularity_bytes;
|
||||
|
||||
ReplicatedMergeTreeTableMetadata() = default;
|
||||
explicit ReplicatedMergeTreeTableMetadata(const MergeTreeData & data);
|
||||
@ -53,6 +53,9 @@ struct ReplicatedMergeTreeTableMetadata
|
||||
};
|
||||
|
||||
Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, bool allow_alter) const;
|
||||
|
||||
private:
|
||||
bool index_granularity_bytes_found_in_zk = false;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -114,7 +114,7 @@ class ClickHouseCluster:
|
||||
cmd += " client"
|
||||
return cmd
|
||||
|
||||
def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None):
|
||||
def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False):
|
||||
"""Add an instance to the cluster.
|
||||
|
||||
name - the name of the instance directory and the value of the 'instance' macro in ClickHouse.
|
||||
@ -134,7 +134,8 @@ class ClickHouseCluster:
|
||||
self, self.base_dir, name, config_dir, main_configs, user_configs, macros, with_zookeeper,
|
||||
self.zookeeper_config_path, with_mysql, with_kafka, with_mongo, self.base_configs_dir, self.server_bin_path,
|
||||
self.odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=hostname,
|
||||
env_variables=env_variables, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address)
|
||||
env_variables=env_variables, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address,
|
||||
with_installed_binary=with_installed_binary)
|
||||
|
||||
self.instances[name] = instance
|
||||
if ipv4_address is not None or ipv6_address is not None:
|
||||
@ -425,11 +426,11 @@ services:
|
||||
image: {image}
|
||||
hostname: {hostname}
|
||||
volumes:
|
||||
- {binary_path}:/usr/bin/clickhouse:ro
|
||||
- {odbc_bridge_bin_path}:/usr/bin/clickhouse-odbc-bridge:ro
|
||||
- {configs_dir}:/etc/clickhouse-server/
|
||||
- {db_dir}:/var/lib/clickhouse/
|
||||
- {logs_dir}:/var/log/clickhouse-server/
|
||||
{binary_volume}
|
||||
{odbc_bridge_volume}
|
||||
{odbc_ini_path}
|
||||
entrypoint: {entrypoint_cmd}
|
||||
cap_add:
|
||||
@ -453,7 +454,7 @@ class ClickHouseInstance:
|
||||
self, cluster, base_path, name, custom_config_dir, custom_main_configs, custom_user_configs, macros,
|
||||
with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_mongo, base_configs_dir, server_bin_path, odbc_bridge_bin_path,
|
||||
clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test",
|
||||
stay_alive=False, ipv4_address=None, ipv6_address=None):
|
||||
stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False):
|
||||
|
||||
self.name = name
|
||||
self.base_cmd = cluster.base_cmd[:]
|
||||
@ -494,6 +495,7 @@ class ClickHouseInstance:
|
||||
self.stay_alive = stay_alive
|
||||
self.ipv4_address = ipv4_address
|
||||
self.ipv6_address = ipv6_address
|
||||
self.with_installed_binary = with_installed_binary
|
||||
|
||||
# Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer
|
||||
def query(self, sql, stdin=None, timeout=None, settings=None, user=None, ignore_error=False):
|
||||
@ -533,7 +535,7 @@ class ClickHouseInstance:
|
||||
|
||||
self.exec_in_container(["bash", "-c", "pkill clickhouse"], user='root')
|
||||
time.sleep(stop_start_wait_sec)
|
||||
self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user='root')
|
||||
self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
|
||||
|
||||
def exec_in_container(self, cmd, detach=False, **kwargs):
|
||||
container = self.get_docker_handle()
|
||||
@ -556,7 +558,16 @@ class ClickHouseInstance:
|
||||
encoded_data = base64.b64encode(data)
|
||||
self.exec_in_container(["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)])
|
||||
|
||||
|
||||
def restart_with_latest_version(self, stop_start_wait_sec=5, callback_onstop=None, signal=15):
|
||||
if not self.stay_alive:
|
||||
raise Exception("Cannot restart not stay alive container")
|
||||
self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root')
|
||||
time.sleep(stop_start_wait_sec)
|
||||
if callback_onstop:
|
||||
callback_onstop(self)
|
||||
self.exec_in_container(["bash", "-c", "cp /usr/share/clickhouse_fresh /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse"], user='root')
|
||||
self.exec_in_container(["bash", "-c", "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], user='root')
|
||||
self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
|
||||
|
||||
def get_docker_handle(self):
|
||||
return self.docker_client.containers.get(self.docker_id)
|
||||
@ -656,6 +667,9 @@ class ClickHouseInstance:
|
||||
if key != "DSN":
|
||||
f.write(key + "=" + value + "\n")
|
||||
|
||||
def replace_config(self, path_to_config, replacement):
|
||||
self.exec_in_container(["bash", "-c", "echo '{}' > {}".format(replacement, path_to_config)])
|
||||
|
||||
def create_dir(self, destroy_dir=True):
|
||||
"""Create the instance directory and all the needed files there."""
|
||||
|
||||
@ -750,13 +764,21 @@ class ClickHouseInstance:
|
||||
if self.ipv6_address is not None:
|
||||
ipv6_address = "ipv6_address: " + self.ipv6_address
|
||||
|
||||
if not self.with_installed_binary:
|
||||
binary_volume = "- " + self.server_bin_path + ":/usr/bin/clickhouse"
|
||||
odbc_bridge_volume = "- " + self.odbc_bridge_bin_path + ":/usr/bin/clickhouse-odbc-bridge"
|
||||
else:
|
||||
binary_volume = "- " + self.server_bin_path + ":/usr/share/clickhouse_fresh"
|
||||
odbc_bridge_volume = "- " + self.odbc_bridge_bin_path + ":/usr/share/clickhouse-odbc-bridge_fresh"
|
||||
|
||||
|
||||
with open(self.docker_compose_path, 'w') as docker_compose:
|
||||
docker_compose.write(DOCKER_COMPOSE_TEMPLATE.format(
|
||||
image=self.image,
|
||||
name=self.name,
|
||||
hostname=self.hostname,
|
||||
binary_path=self.server_bin_path,
|
||||
odbc_bridge_bin_path=self.odbc_bridge_bin_path,
|
||||
binary_volume=binary_volume,
|
||||
odbc_bridge_volume=odbc_bridge_volume,
|
||||
configs_dir=configs_dir,
|
||||
config_d_dir=config_d_dir,
|
||||
db_dir=db_dir,
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
<yandex>
|
||||
<shutdown_wait_unfinished>3</shutdown_wait_unfinished>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<log>/var/log/clickhouse-server/log.log</log>
|
||||
<errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
|
||||
<size>1000M</size>
|
||||
<count>10</count>
|
||||
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
|
||||
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
|
||||
</logger>
|
||||
</yandex>
|
||||
@ -0,0 +1,4 @@
|
||||
<yandex>
|
||||
<merge_tree>
|
||||
</merge_tree>
|
||||
</yandex>
|
||||
@ -0,0 +1,68 @@
|
||||
<yandex>
|
||||
<remote_servers>
|
||||
<test_cluster>
|
||||
<shard>
|
||||
<!-- both latest version -->
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster>
|
||||
<test_cluster>
|
||||
<shard>
|
||||
<!-- 19.6 + latest version -->
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node3</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node4</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster>
|
||||
<test_cluster>
|
||||
<shard>
|
||||
<!-- 19.1 + latest version -->
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node5</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node6</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster>
|
||||
<test_cluster>
|
||||
<shard>
|
||||
<!-- both 19.1 and update to new version -->
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node11</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<default_database>test</default_database>
|
||||
<host>node12</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster>
|
||||
</remote_servers>
|
||||
</yandex>
|
||||
329
dbms/tests/integration/test_adaptive_granularity/test.py
Normal file
329
dbms/tests/integration/test_adaptive_granularity/test.py
Normal file
@ -0,0 +1,329 @@
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from multiprocessing.dummy import Pool
|
||||
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
|
||||
|
||||
from helpers.test_tools import assert_eq_with_retry
|
||||
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
node1 = cluster.add_instance('node1', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
|
||||
node2 = cluster.add_instance('node2', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
|
||||
|
||||
node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', with_installed_binary=True)
|
||||
node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
|
||||
|
||||
node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', with_installed_binary=True)
|
||||
node6 = cluster.add_instance('node6', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
|
||||
|
||||
node7 = cluster.add_instance('node7', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', stay_alive=True, with_installed_binary=True)
|
||||
node8 = cluster.add_instance('node8', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True)
|
||||
|
||||
node9 = cluster.add_instance('node9', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True)
|
||||
node10 = cluster.add_instance('node10', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', stay_alive=True, with_installed_binary=True)
|
||||
|
||||
node11 = cluster.add_instance('node11', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True)
|
||||
node12 = cluster.add_instance('node12', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True)
|
||||
|
||||
|
||||
def prepare_single_pair_with_setting(first_node, second_node, group):
|
||||
for node in (first_node, second_node):
|
||||
node.query("CREATE DATABASE IF NOT EXISTS test")
|
||||
|
||||
# Two tables with adaptive granularity
|
||||
first_node.query(
|
||||
'''
|
||||
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 10485760
|
||||
'''.format(g=group))
|
||||
|
||||
second_node.query(
|
||||
'''
|
||||
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 10485760
|
||||
'''.format(g=group))
|
||||
|
||||
# Two tables with fixed granularity
|
||||
first_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 0
|
||||
'''.format(g=group))
|
||||
|
||||
second_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 0
|
||||
'''.format(g=group))
|
||||
|
||||
# Two tables with different granularity
|
||||
with pytest.raises(QueryRuntimeException):
|
||||
first_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 10485760
|
||||
'''.format(g=group))
|
||||
|
||||
second_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 0
|
||||
'''.format(g=group))
|
||||
|
||||
# Two tables with different granularity, but enabled mixed parts
|
||||
first_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1
|
||||
'''.format(g=group))
|
||||
|
||||
second_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1
|
||||
'''.format(g=group))
|
||||
|
||||
|
||||
def prepare_single_pair_without_setting(first_node, second_node, group):
|
||||
for node in (first_node, second_node):
|
||||
node.query("CREATE DATABASE IF NOT EXISTS test")
|
||||
|
||||
# Two tables with fixed granularity
|
||||
first_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
'''.format(g=group))
|
||||
|
||||
second_node.query(
|
||||
'''
|
||||
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes = 0
|
||||
'''.format(g=group))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def start_static_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
prepare_single_pair_with_setting(node1, node2, "shard1")
|
||||
prepare_single_pair_with_setting(node3, node4, "shard2")
|
||||
prepare_single_pair_without_setting(node5, node6, "shard3")
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('first_node', 'second_node', 'table'),
|
||||
[
|
||||
(node1, node2, 'table_by_default'),
|
||||
(node1, node2, 'table_with_fixed_granularity'),
|
||||
(node3, node4, 'table_by_default'),
|
||||
(node3, node4, 'table_with_fixed_granularity'),
|
||||
(node5, node6, 'table_with_fixed_granularity'),
|
||||
]
|
||||
)
|
||||
def test_different_versions_cluster(start_static_cluster, first_node, second_node, table):
|
||||
counter = 1
|
||||
for n1, n2 in ((first_node, second_node), (second_node, first_node)):
|
||||
n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(tbl=table, c1=counter * 2, c2=counter * 2 + 1))
|
||||
n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
|
||||
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
||||
n1.query("DETACH TABLE {tbl}".format(tbl=table))
|
||||
n2.query("DETACH TABLE {tbl}".format(tbl=table))
|
||||
n1.query("ATTACH TABLE {tbl}".format(tbl=table))
|
||||
n2.query("ATTACH TABLE {tbl}".format(tbl=table))
|
||||
assert_eq_with_retry(n1, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
||||
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
||||
n1.query("OPTIMIZE TABLE {tbl} FINAL".format(tbl=table))
|
||||
n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
|
||||
assert_eq_with_retry(n1, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
||||
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
||||
counter += 1
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def start_dynamic_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
node7.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node7.query(
|
||||
'''
|
||||
CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity_bytes=10485760
|
||||
''')
|
||||
|
||||
node8.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node9.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node10.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node11.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node12.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('n', 'tables'),
|
||||
[
|
||||
(node7, ['table_with_default_granularity', 'table_with_adaptive_default_granularity']),
|
||||
(node8, ['table_with_default_granularity']),
|
||||
]
|
||||
)
|
||||
def test_version_single_node_update(start_dynamic_cluster, n, tables):
|
||||
for table in tables:
|
||||
n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table))
|
||||
n.restart_with_latest_version()
|
||||
for table in tables:
|
||||
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n'
|
||||
n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table))
|
||||
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n'
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('node',),
|
||||
[
|
||||
(node9,),
|
||||
(node10,)
|
||||
]
|
||||
)
|
||||
def test_mixed_granularity_single_node(start_dynamic_cluster, node):
|
||||
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
||||
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)")
|
||||
|
||||
def callback(n):
|
||||
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
|
||||
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
|
||||
|
||||
node.restart_with_latest_version(callback_onstop=callback)
|
||||
node.query("SYSTEM RELOAD CONFIG")
|
||||
assert_eq_with_retry(node, "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'", '1')
|
||||
assert node.query("SELECT count() from table_with_default_granularity") == '4\n'
|
||||
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
||||
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
||||
node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL")
|
||||
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
||||
path_to_merged_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
|
||||
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_to_merged_part)]) # check that we have adaptive files
|
||||
|
||||
path_to_old_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip()
|
||||
|
||||
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(p=path_to_old_part)]) # check that we have non adaptive files
|
||||
|
||||
def test_version_update_two_nodes(start_dynamic_cluster):
|
||||
node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
||||
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity")
|
||||
assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n'
|
||||
node12.restart_with_latest_version()
|
||||
node12.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
||||
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity")
|
||||
assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n'
|
||||
|
||||
node12.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node11.query(
|
||||
'''
|
||||
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1')
|
||||
PARTITION BY toYYYYMM(date)
|
||||
ORDER BY id
|
||||
''')
|
||||
|
||||
node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
||||
with pytest.raises(QueryTimeoutExceedException):
|
||||
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=5)
|
||||
node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
||||
|
||||
node11.restart_with_latest_version(signal=9) # just to be sure
|
||||
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=5)
|
||||
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=5)
|
||||
node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
|
||||
node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
|
||||
|
||||
node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)")
|
||||
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity")
|
||||
assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '6\n'
|
||||
Loading…
Reference in New Issue
Block a user