thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

added test with ttl move to volume

Browse Source
This commit is contained in:
Nikita Mikhaylov 2021-04-23 17:39:03 +03:00
parent 9022646633
commit 267319ffeb
3 changed files with 159 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
tests/integration/test_cluster_copier/configs_two_nodes/conf.d/storage_configuration.xml Normal file
View File

@ -0,0 +1,34 @@
<yandex>
<storage_configuration>
<disks>
<default>
</default>
<jbod1>
<path>/jbod1/</path>
</jbod1>
<jbod2>
<path>/jbod2/</path>
</jbod2>
<external>
<path>/external/</path>
</external>
</disks>
<policies>
<external_with_jbods>
<volumes>
<external>
<disk>external</disk>
</external>
<main>
<disk>jbod1</disk>
<disk>jbod2</disk>
</main>
</volumes>
</external_with_jbods>
</policies>
</storage_configuration>
</yandex>

40
tests/integration/test_cluster_copier/task_ttl_move_to_volume.xml Normal file
View File

@ -0,0 +1,40 @@
<?xml version="1.0"?>
<yandex>
<remote_servers>
<source>
<shard>
<internal_replication>false</internal_replication>
<replica>
<host>first</host>
<port>9000</port>
</replica>
</shard>
</source>
<destination>
<shard>
<internal_replication>false</internal_replication>
<replica>
<host>second</host>
<port>9000</port>
</replica>
</shard>
</destination>
</remote_servers>
<max_workers>2</max_workers>
<tables>
<table_events>
<cluster_pull>source</cluster_pull>
<database_pull>db_move_to_volume</database_pull>
<table_pull>source</table_pull>
<cluster_push>destination</cluster_push>
<database_push>db_move_to_volume</database_push>
<table_push>destination</table_push>
<engine>ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' SETTINGS storage_policy = 'external_with_jbods'</engine>
<sharding_key>rand()</sharding_key>
</table_events>
</tables>
</yandex>

88
tests/integration/test_cluster_copier/test_two_nodes.py
View File

@ -22,11 +22,22 @@ def started_cluster():
cluster = ClickHouseCluster(__file__)
for name in ["first", "second"]:
cluster.add_instance(name,
main_configs=["configs_two_nodes/conf.d/clusters.xml", "configs_two_nodes/conf.d/ddl.xml"], user_configs=["configs_two_nodes/users.xml"],
instance = cluster.add_instance(name,
main_configs=[
"configs_two_nodes/conf.d/clusters.xml",
"configs_two_nodes/conf.d/ddl.xml",
"configs_two_nodes/conf.d/storage_configuration.xml"],
user_configs=["configs_two_nodes/users.xml"],
with_zookeeper=True, external_data_path=os.path.join(CURRENT_TEST_DIR, "./data"))
cluster.start()
for name in ["first", "second"]:
instance = cluster.instances[name]
instance.exec_in_container(['bash', '-c', 'mkdir /jbod1'])
instance.exec_in_container(['bash', '-c', 'mkdir /jbod2'])
instance.exec_in_container(['bash', '-c', 'mkdir /external'])
yield cluster
finally:
@ -249,6 +260,72 @@ class TaskSkipIndex:
assert a == b, "Data"
class TaskTTLMoveToVolume:
def __init__(self, cluster):
self.cluster = cluster
self.zk_task_path = '/clickhouse-copier/task_ttl_move_to_volume'
self.container_task_file = "/task_ttl_move_to_volume.xml"
for instance_name, _ in cluster.instances.items():
instance = cluster.instances[instance_name]
instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_ttl_move_to_volume.xml'), self.container_task_file)
print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file))
def start(self):
first = cluster.instances["first"]
first.query("CREATE DATABASE db_move_to_volume;")
first.query("""CREATE TABLE db_move_to_volume.source
(
Column1 UInt64,
Column2 Int32,
Column3 Date,
Column4 DateTime,
Column5 String
)
ENGINE = MergeTree()
PARTITION BY (toYYYYMMDD(Column3), Column3)
PRIMARY KEY (Column1, Column2, Column3)
ORDER BY (Column1, Column2, Column3)
TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external'
SETTINGS storage_policy = 'external_with_jbods';""")
first.query("""INSERT INTO db_move_to_volume.source SELECT * FROM generateRandom(
'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""")
second = cluster.instances["second"]
second.query("CREATE DATABASE db_move_to_volume;")
second.query("""CREATE TABLE db_move_to_volume.destination
(
Column1 UInt64,
Column2 Int32,
Column3 Date,
Column4 DateTime,
Column5 String
) ENGINE = MergeTree()
PARTITION BY toYYYYMMDD(Column3)
ORDER BY (Column3, Column2, Column1)
TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external'
SETTINGS storage_policy = 'external_with_jbods';""")
print("Preparation completed")
def check(self):
first = cluster.instances["first"]
second = cluster.instances["second"]
a = first.query("SELECT count() from db_move_to_volume.source")
b = second.query("SELECT count() from db_move_to_volume.destination")
assert a == b, "Count"
a = TSV(first.query("""SELECT sipHash64(*) from db_move_to_volume.source
ORDER BY (Column1, Column2, Column3, Column4, Column5)"""))
b = TSV(second.query("""SELECT sipHash64(*) from db_move_to_volume.destination
ORDER BY (Column1, Column2, Column3, Column4, Column5)"""))
assert a == b, "Data"
def execute_task(task, cmd_options):
task.start()
@ -311,4 +388,9 @@ def test_ttl_columns(started_cluster):
@pytest.mark.timeout(600)
def test_skip_index(started_cluster):
execute_task(TaskSkipIndex(started_cluster), [])
execute_task(TaskSkipIndex(started_cluster), [])
@pytest.mark.timeout(600)
def test_ttl_move_to_volume(started_cluster):
execute_task(TaskTTLMoveToVolume(started_cluster), [])