thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use absolute paths to backup locations (shadow)

Browse Source 
Tests are frankenstain. Had to hide absolute path `/tmp/clickhouse.test..IP69I/data/shadow/test_01417_single_part_7/data/default/table_for_freeze/7_8_8_0` to make them deterministic and independend of the method they are run (local vs remote target)
This commit is contained in:
Nicolae Vartolomei 2020-07-30 18:03:26 +01:00
parent c1b71ab06e
commit 85174d64a0
11 changed files with 182 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -3354,7 +3354,8 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m
result.push_back(PartitionCommandResultInfo{ result.push_back(PartitionCommandResultInfo{
.partition_id = part->info.partition_id, .partition_id = part->info.partition_id,
.part_name = part->name, .part_name = part->name,
.backup_path = backup_path, .backup_path = part->volume->getDisk()->getPath() + backup_path,
.part_backup_path = part->volume->getDisk()->getPath() + backup_part_path,
.backup_name = backup_name, .backup_name = backup_name,
}); });
++parts_processed; ++parts_processed;

7
src/Storages/PartitionCommands.cpp
View File

@ -153,6 +153,8 @@ Pipes convertCommandsResultToSource(const PartitionCommandsResultInfo & commands
if (!command_result.backup_path.empty() && !header.has("backup_path")) if (!command_result.backup_path.empty() && !header.has("backup_path"))
header.insert(ColumnWithTypeAndName(std::make_shared<DataTypeString>(), "backup_path")); header.insert(ColumnWithTypeAndName(std::make_shared<DataTypeString>(), "backup_path"));
if (!command_result.backup_path.empty() && !header.has("part_backup_path"))
header.insert(ColumnWithTypeAndName(std::make_shared<DataTypeString>(), "part_backup_path"));
} }
MutableColumns res_columns = header.cloneEmptyColumns(); MutableColumns res_columns = header.cloneEmptyColumns();
@ -177,6 +179,11 @@ Pipes convertCommandsResultToSource(const PartitionCommandsResultInfo & commands
size_t pos = header.getPositionByName("backup_path"); size_t pos = header.getPositionByName("backup_path");
res_columns[pos]->insert(command_result.backup_path); res_columns[pos]->insert(command_result.backup_path);
} }
if (header.has("part_backup_path"))
{
size_t pos = header.getPositionByName("part_backup_path");
res_columns[pos]->insert(command_result.part_backup_path);
}
} }
Chunk chunk(std::move(res_columns), commands_result.size()); Chunk chunk(std::move(res_columns), commands_result.size());

4
src/Storages/PartitionCommands.h
View File

@ -89,8 +89,10 @@ struct PartitionCommandResultInfo
String part_name; String part_name;
/// Part name in /detached directory, filled in ATTACH /// Part name in /detached directory, filled in ATTACH
String old_part_name; String old_part_name;
/// Path to backup directory, filled in FREEZE /// Absolute path to backup directory, filled in FREEZE
String backup_path; String backup_path;
/// Absolute path part backup, filled in FREEZE
String part_backup_path;
/// Name of the backup (specified by user or increment value), filled in /// Name of the backup (specified by user or increment value), filled in
/// FREEZE /// FREEZE
String backup_name; String backup_name;

0
tests/integration/test_freeze_table/__init__.py Normal file
View File

63
tests/integration/test_freeze_table/test.py Normal file
View File

@ -0,0 +1,63 @@
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__)
node = cluster.add_instance("node")
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def test_freeze_table(started_cluster):
node.query('''
CREATE TABLE table_for_freeze
(
key UInt64,
value String
)
ENGINE = MergeTree()
ORDER BY key
PARTITION BY key % 10;
''')
node.query('''
INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10);
''')
freeze_result = TSV.toMat(node.query('''
ALTER TABLE table_for_freeze
FREEZE WITH NAME 'test_01417'
FORMAT TSVWithNames
SETTINGS alter_partition_verbose_result = 1;
'''))
assert 11 == len(freeze_result)
path_col_ix = freeze_result[0].index('part_backup_path')
for row in freeze_result[1:]: # skip header
part_backup_path = row[path_col_ix]
node.exec_in_container(
["bash", "-c", "test -d {}".format(part_backup_path)]
)
freeze_result = TSV.toMat(node.query('''
ALTER TABLE table_for_freeze
FREEZE PARTITION '3'
WITH NAME
'test_01417_single_part'
FORMAT TSVWithNames
SETTINGS alter_partition_verbose_result = 1;
'''))
assert 2 == len(freeze_result)
path_col_ix = freeze_result[0].index('part_backup_path')
for row in freeze_result[1:]: # skip header
part_backup_path = row[path_col_ix]
assert 'test_01417_single_part' in part_backup_path
node.exec_in_container(
["bash", "-c", "test -d {}".format(part_backup_path)]
)

32
tests/queries/0_stateless/01417_freeze_partition_verbose.reference
View File

@ -1,18 +1,18 @@
command_type partition_id part_name backup_name backup_path command_type partition_id part_name backup_name
FREEZE ALL 0 0_1_1_0 test_01417 shadow/test_01417/ FREEZE ALL 0 0_1_1_0 test_01417
FREEZE ALL 1 1_2_2_0 test_01417 shadow/test_01417/ FREEZE ALL 1 1_2_2_0 test_01417
FREEZE ALL 2 2_3_3_0 test_01417 shadow/test_01417/ FREEZE ALL 2 2_3_3_0 test_01417
FREEZE ALL 3 3_4_4_0 test_01417 shadow/test_01417/ FREEZE ALL 3 3_4_4_0 test_01417
FREEZE ALL 4 4_5_5_0 test_01417 shadow/test_01417/ FREEZE ALL 4 4_5_5_0 test_01417
FREEZE ALL 5 5_6_6_0 test_01417 shadow/test_01417/ FREEZE ALL 5 5_6_6_0 test_01417
FREEZE ALL 6 6_7_7_0 test_01417 shadow/test_01417/ FREEZE ALL 6 6_7_7_0 test_01417
FREEZE ALL 7 7_8_8_0 test_01417 shadow/test_01417/ FREEZE ALL 7 7_8_8_0 test_01417
FREEZE ALL 8 8_9_9_0 test_01417 shadow/test_01417/ FREEZE ALL 8 8_9_9_0 test_01417
FREEZE ALL 9 9_10_10_0 test_01417 shadow/test_01417/ FREEZE ALL 9 9_10_10_0 test_01417
command_type partition_id part_name backup_name backup_path command_type partition_id part_name backup_name
FREEZE PARTITION 3 3_4_4_0 test_01417_single_part shadow/test_01417_single_part/ FREEZE PARTITION 3 3_4_4_0 test_01417_single_part
command_type partition_id part_name old_part_name command_type partition_id part_name old_part_name
ATTACH PARTITION 3 3_12_12_0 3_4_4_0 ATTACH PARTITION 3 3_12_12_0 3_4_4_0
command_type partition_id part_name backup_name backup_path old_part_name command_type partition_id part_name backup_name old_part_name
FREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7 shadow/test_01417_single_part_7/ FREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7
ATTACH PART 5 5_13_13_0 5_6_6_0 ATTACH PART 5 5_13_13_0 5_6_6_0

38
tests/queries/0_stateless/01417_freeze_partition_verbose.sh Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String'
ATTACH_OUT_STRUCTURE='old_part_name String'
FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path String'
# setup
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;"
${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze (key UInt64, value String) ENGINE = MergeTree() ORDER BY key PARTITION BY key % 10;"
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10);"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze DETACH PARTITION '3';"
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze VALUES (3, '3');"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, old_part_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze DETACH PARTITION '5';"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_6_6_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name, old_part_name FROM table"
# teardown
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;"

28
tests/queries/0_stateless/01417_freeze_partition_verbose.sql
View File

@ -1,28 +0,0 @@
DROP TABLE IF EXISTS table_for_freeze;
CREATE TABLE table_for_freeze
(
key UInt64,
value String
)
ENGINE = MergeTree()
ORDER BY key
PARTITION BY key % 10;
INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10);
ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze DETACH PARTITION '3';
INSERT INTO table_for_freeze VALUES (3, '3');
ALTER TABLE table_for_freeze ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze DETACH PARTITION '5';
ALTER TABLE table_for_freeze FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_6_6_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
DROP TABLE IF EXISTS table_for_freeze;

30
tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.reference
View File

@ -1,18 +1,18 @@
command_type partition_id part_name backup_name backup_path command_type partition_id part_name backup_name
FREEZE ALL 0 0_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 0 0_0_0_0 test_01417
FREEZE ALL 1 1_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 1 1_0_0_0 test_01417
FREEZE ALL 2 2_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 2 2_0_0_0 test_01417
FREEZE ALL 3 3_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 3 3_0_0_0 test_01417
FREEZE ALL 4 4_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 4 4_0_0_0 test_01417
FREEZE ALL 5 5_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 5 5_0_0_0 test_01417
FREEZE ALL 6 6_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 6 6_0_0_0 test_01417
FREEZE ALL 7 7_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 7 7_0_0_0 test_01417
FREEZE ALL 8 8_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 8 8_0_0_0 test_01417
FREEZE ALL 9 9_0_0_0 test_01417 shadow/test_01417/ FREEZE ALL 9 9_0_0_0 test_01417
command_type partition_id part_name backup_name backup_path command_type partition_id part_name backup_name
FREEZE PARTITION 3 3_0_0_0 test_01417_single_part shadow/test_01417_single_part/ FREEZE PARTITION 3 3_0_0_0 test_01417_single_part
command_type partition_id part_name old_part_name command_type partition_id part_name old_part_name
ATTACH PARTITION 3 3_3_3_0 3_0_0_0 ATTACH PARTITION 3 3_3_3_0 3_0_0_0
command_type partition_id part_name backup_name backup_path old_part_name command_type partition_id part_name backup_name old_part_name
FREEZE PARTITION 7 7_0_0_0 test_01417_single_part_7 shadow/test_01417_single_part_7/ FREEZE PARTITION 7 7_0_0_0 test_01417_single_part_7
ATTACH PART 5 5_2_2_0 5_0_0_0 ATTACH PART 5 5_2_2_0 5_0_0_0

38
tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
ALTER_OUT_STRUCTURE='command_type String, partition_id String, part_name String'
ATTACH_OUT_STRUCTURE='old_part_name String'
FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path String'
# setup
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;"
${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;"
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10);"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated DETACH PARTITION '3';"
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated VALUES (3, '3');"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, old_part_name FROM table"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated DETACH PARTITION '5';"
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_0_0_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \
--query "SELECT command_type, partition_id, part_name, backup_name, old_part_name FROM table"
# teardown
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;"

28
tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sql
View File

@ -1,28 +0,0 @@
DROP TABLE IF EXISTS table_for_freeze_replicated;
CREATE TABLE table_for_freeze_replicated
(
key UInt64,
value String
)
ENGINE = ReplicatedMergeTree('/test/table_for_freeze_replicated', '1')
ORDER BY key
PARTITION BY key % 10;
INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10);
ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze_replicated DETACH PARTITION '3';
INSERT INTO table_for_freeze_replicated VALUES (3, '3');
ALTER TABLE table_for_freeze_replicated ATTACH PARTITION '3' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
ALTER TABLE table_for_freeze_replicated DETACH PARTITION '5';
ALTER TABLE table_for_freeze_replicated FREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7', ATTACH PART '5_0_0_0' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;
DROP TABLE IF EXISTS table_for_freeze_replicated;