diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 740d44605ee..6c4ad3d7eab 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3354,7 +3354,8 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m result.push_back(PartitionCommandResultInfo{ .partition_id = part->info.partition_id, .part_name = part->name, - .backup_path = backup_path, + .backup_path = part->volume->getDisk()->getPath() + backup_path, + .part_backup_path = part->volume->getDisk()->getPath() + backup_part_path, .backup_name = backup_name, }); ++parts_processed; diff --git a/src/Storages/PartitionCommands.cpp b/src/Storages/PartitionCommands.cpp index e3f542695cb..bb83e6860f7 100644 --- a/src/Storages/PartitionCommands.cpp +++ b/src/Storages/PartitionCommands.cpp @@ -153,6 +153,8 @@ Pipes convertCommandsResultToSource(const PartitionCommandsResultInfo & commands if (!command_result.backup_path.empty() && !header.has("backup_path")) header.insert(ColumnWithTypeAndName(std::make_shared(), "backup_path")); + if (!command_result.backup_path.empty() && !header.has("part_backup_path")) + header.insert(ColumnWithTypeAndName(std::make_shared(), "part_backup_path")); } MutableColumns res_columns = header.cloneEmptyColumns(); @@ -177,6 +179,11 @@ Pipes convertCommandsResultToSource(const PartitionCommandsResultInfo & commands size_t pos = header.getPositionByName("backup_path"); res_columns[pos]->insert(command_result.backup_path); } + if (header.has("part_backup_path")) + { + size_t pos = header.getPositionByName("part_backup_path"); + res_columns[pos]->insert(command_result.part_backup_path); + } } Chunk chunk(std::move(res_columns), commands_result.size()); diff --git a/src/Storages/PartitionCommands.h b/src/Storages/PartitionCommands.h index 9f5540fdbdf..ea289b5c03a 100644 --- a/src/Storages/PartitionCommands.h +++ b/src/Storages/PartitionCommands.h @@ -89,8 +89,10 @@ struct PartitionCommandResultInfo String part_name; /// Part name in /detached directory, filled in ATTACH String old_part_name; - /// Path to backup directory, filled in FREEZE + /// Absolute path to backup directory, filled in FREEZE String backup_path; + /// Absolute path part backup, filled in FREEZE + String part_backup_path; /// Name of the backup (specified by user or increment value), filled in /// FREEZE String backup_name; diff --git a/tests/integration/test_freeze_table/__init__.py b/tests/integration/test_freeze_table/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_freeze_table/test.py b/tests/integration/test_freeze_table/test.py new file mode 100644 index 00000000000..ab1af3f371e --- /dev/null +++ b/tests/integration/test_freeze_table/test.py @@ -0,0 +1,63 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_freeze_table(started_cluster): + node.query(''' + CREATE TABLE table_for_freeze + ( + key UInt64, + value String + ) + ENGINE = MergeTree() + ORDER BY key + PARTITION BY key % 10; + ''') + node.query(''' + INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10); + ''') + + freeze_result = TSV.toMat(node.query(''' + ALTER TABLE table_for_freeze + FREEZE WITH NAME 'test_01417' + FORMAT TSVWithNames + SETTINGS alter_partition_verbose_result = 1; + ''')) + assert 11 == len(freeze_result) + path_col_ix = freeze_result[0].index('part_backup_path') + for row in freeze_result[1:]: # skip header + part_backup_path = row[path_col_ix] + node.exec_in_container( + ["bash", "-c", "test -d {}".format(part_backup_path)] + ) + + freeze_result = TSV.toMat(node.query(''' + ALTER TABLE table_for_freeze + FREEZE PARTITION '3' + WITH NAME + 'test_01417_single_part' + FORMAT TSVWithNames + SETTINGS alter_partition_verbose_result = 1; + ''')) + assert 2 == len(freeze_result) + path_col_ix = freeze_result[0].index('part_backup_path') + for row in freeze_result[1:]: # skip header + part_backup_path = row[path_col_ix] + assert 'test_01417_single_part' in part_backup_path + node.exec_in_container( + ["bash", "-c", "test -d {}".format(part_backup_path)] + ) diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.reference b/tests/queries/0_stateless/01417_freeze_partition_verbose.reference index a3be410532b..e648f619c50 100644 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose.reference +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose.reference @@ -1,18 +1,18 @@ -command_type partition_id part_name backup_name backup_path -FREEZE ALL 0 0_1_1_0 test_01417 shadow/test_01417/ -FREEZE ALL 1 1_2_2_0 test_01417 shadow/test_01417/ -FREEZE ALL 2 2_3_3_0 test_01417 shadow/test_01417/ -FREEZE ALL 3 3_4_4_0 test_01417 shadow/test_01417/ -FREEZE ALL 4 4_5_5_0 test_01417 shadow/test_01417/ -FREEZE ALL 5 5_6_6_0 test_01417 shadow/test_01417/ -FREEZE ALL 6 6_7_7_0 test_01417 shadow/test_01417/ -FREEZE ALL 7 7_8_8_0 test_01417 shadow/test_01417/ -FREEZE ALL 8 8_9_9_0 test_01417 shadow/test_01417/ -FREEZE ALL 9 9_10_10_0 test_01417 shadow/test_01417/ -command_type partition_id part_name backup_name backup_path -FREEZE PARTITION 3 3_4_4_0 test_01417_single_part shadow/test_01417_single_part/ +command_type partition_id part_name backup_name +FREEZE ALL 0 0_1_1_0 test_01417 +FREEZE ALL 1 1_2_2_0 test_01417 +FREEZE ALL 2 2_3_3_0 test_01417 +FREEZE ALL 3 3_4_4_0 test_01417 +FREEZE ALL 4 4_5_5_0 test_01417 +FREEZE ALL 5 5_6_6_0 test_01417 +FREEZE ALL 6 6_7_7_0 test_01417 +FREEZE ALL 7 7_8_8_0 test_01417 +FREEZE ALL 8 8_9_9_0 test_01417 +FREEZE ALL 9 9_10_10_0 test_01417 +command_type partition_id part_name backup_name +FREEZE PARTITION 3 3_4_4_0 test_01417_single_part command_type partition_id part_name old_part_name ATTACH PARTITION 3 3_12_12_0 3_4_4_0 -command_type partition_id part_name backup_name backup_path old_part_name -FREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7 shadow/test_01417_single_part_7/ -ATTACH PART 5 5_13_13_0 5_6_6_0 +command_type partition_id part_name backup_name old_part_name +FREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7 +ATTACH PART 5 5_13_13_0 5_6_6_0 diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh new file mode 100755 index 00000000000..f15eb26e4b8 --- /dev/null +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String' +ATTACH_OUT_STRUCTURE='old_part_name String' +FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path String' + +# setup +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze (key UInt64, value String) ENGINE = MergeTree() ORDER BY key PARTITION BY key % 10;" +${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10);" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name FROM table" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name FROM table" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze DETACH PARTITION '3';" +${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze VALUES (3, '3');" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, old_part_name FROM table" + + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze DETACH PARTITION '5';" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_6_6_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name, old_part_name FROM table" + +# teardown +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;" diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.sql b/tests/queries/0_stateless/01417_freeze_partition_verbose.sql deleted file mode 100644 index 176093b4323..00000000000 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose.sql +++ /dev/null @@ -1,28 +0,0 @@ -DROP TABLE IF EXISTS table_for_freeze; - -CREATE TABLE table_for_freeze -( - key UInt64, - value String -) -ENGINE = MergeTree() -ORDER BY key -PARTITION BY key % 10; - -INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10); - -ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze DETACH PARTITION '3'; - -INSERT INTO table_for_freeze VALUES (3, '3'); - -ALTER TABLE table_for_freeze ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze DETACH PARTITION '5'; - -ALTER TABLE table_for_freeze FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_6_6_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -DROP TABLE IF EXISTS table_for_freeze; diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.reference b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.reference index 7fea72f847d..086a60de715 100644 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.reference +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.reference @@ -1,18 +1,18 @@ -command_type partition_id part_name backup_name backup_path -FREEZE ALL 0 0_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 1 1_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 2 2_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 3 3_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 4 4_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 5 5_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 6 6_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 7 7_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 8 8_0_0_0 test_01417 shadow/test_01417/ -FREEZE ALL 9 9_0_0_0 test_01417 shadow/test_01417/ -command_type partition_id part_name backup_name backup_path -FREEZE PARTITION 3 3_0_0_0 test_01417_single_part shadow/test_01417_single_part/ +command_type partition_id part_name backup_name +FREEZE ALL 0 0_0_0_0 test_01417 +FREEZE ALL 1 1_0_0_0 test_01417 +FREEZE ALL 2 2_0_0_0 test_01417 +FREEZE ALL 3 3_0_0_0 test_01417 +FREEZE ALL 4 4_0_0_0 test_01417 +FREEZE ALL 5 5_0_0_0 test_01417 +FREEZE ALL 6 6_0_0_0 test_01417 +FREEZE ALL 7 7_0_0_0 test_01417 +FREEZE ALL 8 8_0_0_0 test_01417 +FREEZE ALL 9 9_0_0_0 test_01417 +command_type partition_id part_name backup_name +FREEZE PARTITION 3 3_0_0_0 test_01417_single_part command_type partition_id part_name old_part_name ATTACH PARTITION 3 3_3_3_0 3_0_0_0 -command_type partition_id part_name backup_name backup_path old_part_name -FREEZE PARTITION 7 7_0_0_0 test_01417_single_part_7 shadow/test_01417_single_part_7/ +command_type partition_id part_name backup_name old_part_name +FREEZE PARTITION 7 7_0_0_0 test_01417_single_part_7 ATTACH PART 5 5_2_2_0 5_0_0_0 diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh new file mode 100644 index 00000000000..9e255ff2548 --- /dev/null +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String' +ATTACH_OUT_STRUCTURE='old_part_name String' +FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path String' + +# setup + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;" +${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10);" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name FROM table" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name FROM table" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated DETACH PARTITION '3';" +${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated VALUES (3, '3');" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, old_part_name FROM table" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated DETACH PARTITION '5';" + +${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_0_0_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ + | ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \ + --query "SELECT command_type, partition_id, part_name, backup_name, old_part_name FROM table" + +# teardown +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;" diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sql b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sql deleted file mode 100644 index 4947d0c9fd8..00000000000 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sql +++ /dev/null @@ -1,28 +0,0 @@ -DROP TABLE IF EXISTS table_for_freeze_replicated; - -CREATE TABLE table_for_freeze_replicated -( - key UInt64, - value String -) -ENGINE = ReplicatedMergeTree('/test/table_for_freeze_replicated', '1') -ORDER BY key -PARTITION BY key % 10; - -INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10); - -ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze_replicated DETACH PARTITION '3'; - -INSERT INTO table_for_freeze_replicated VALUES (3, '3'); - -ALTER TABLE table_for_freeze_replicated ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -ALTER TABLE table_for_freeze_replicated DETACH PARTITION '5'; - -ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_0_0_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - -DROP TABLE IF EXISTS table_for_freeze_replicated;