add tests and comments

This commit is contained in:
Pavel Kruglov 2020-10-13 21:25:45 +03:00
parent 6d9b310f61
commit 1ccf10505a
6 changed files with 41 additions and 3 deletions

View File

@ -373,6 +373,8 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition(
return false; return false;
} }
/// If final, optimize_skip_merged_partitions is true and we have only one part in partition with level > 0
/// than we don't select it to merge
if (final && data.getSettings()->optimize_skip_merged_partitions && parts.size() == 1 && parts[0]->info.level > 0) if (final && data.getSettings()->optimize_skip_merged_partitions && parts.size() == 1 && parts[0]->info.level > 0)
{ {
*is_single_merged_part = true; *is_single_merged_part = true;
@ -637,6 +639,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
{ {
static const String TMP_PREFIX = "tmp_merge_"; static const String TMP_PREFIX = "tmp_merge_";
if (merges_blocker.isCancelled()) if (merges_blocker.isCancelled())
throw Exception("Cancelled merging parts", ErrorCodes::ABORTED); throw Exception("Cancelled merging parts", ErrorCodes::ABORTED);

View File

@ -88,7 +88,9 @@ public:
String * out_disable_reason = nullptr); String * out_disable_reason = nullptr);
/** Select all the parts in the specified partition for merge, if possible. /** Select all the parts in the specified partition for merge, if possible.
* final - choose to merge even a single part - that is, allow to merge one part "with itself". * final - choose to merge even a single part - that is, allow to merge one part "with itself",
* but if setting optimize_skip_merged_partitions is true (it's true as default) than single part with level > 0
* won't be merged with itself.
*/ */
bool selectAllPartsToMergeWithinPartition( bool selectAllPartsToMergeWithinPartition(
FutureMergedMutatedPart & future_part, FutureMergedMutatedPart & future_part,

View File

@ -654,6 +654,9 @@ bool StorageMergeTree::merge(
}; };
bool selected = false; bool selected = false;
/// This flag is true when there is only one part in partition, it's level > 0
/// and setting optimize_skip_merged_partitions is true
bool is_single_merged_part = false; bool is_single_merged_part = false;
if (partition_id.empty()) if (partition_id.empty())
@ -710,6 +713,7 @@ bool StorageMergeTree::merge(
if (!selected) if (!selected)
{ {
/// If is_single_merged_part is true we treat this part as already merged
if (final && is_single_merged_part) if (final && is_single_merged_part)
{ {
return true; return true;

View File

@ -3761,12 +3761,16 @@ bool StorageReplicatedMergeTree::optimize(
ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper);
FutureMergedMutatedPart future_merged_part; FutureMergedMutatedPart future_merged_part;
/// This flag is true when there is only one part in partition, it's level > 0
/// and setting optimize_skip_merged_partitions is true
bool is_single_merged_part = false; bool is_single_merged_part = false;
bool selected = merger_mutator.selectAllPartsToMergeWithinPartition( bool selected = merger_mutator.selectAllPartsToMergeWithinPartition(
future_merged_part, disk_space, can_merge, partition_id, true, &is_single_merged_part, nullptr); future_merged_part, disk_space, can_merge, partition_id, true, &is_single_merged_part, nullptr);
if (!selected) if (!selected)
{ {
/// If is_single_merged_part is true we treat this part as already merged
if (is_single_merged_part) if (is_single_merged_part)
return true; return true;
break; break;
@ -3803,6 +3807,9 @@ bool StorageReplicatedMergeTree::optimize(
FutureMergedMutatedPart future_merged_part; FutureMergedMutatedPart future_merged_part;
String disable_reason; String disable_reason;
bool selected = false; bool selected = false;
/// This flag is true when there is only one part in partition, it's level > 0
/// and setting optimize_skip_merged_partitions is true
bool is_single_merged_part = false; bool is_single_merged_part = false;
if (!partition) if (!partition)
{ {
@ -3819,6 +3826,7 @@ bool StorageReplicatedMergeTree::optimize(
if (!selected) if (!selected)
{ {
/// If is_single_merged_part is true we treat this part as already merged
if (final && is_single_merged_part) if (final && is_single_merged_part)
return true; return true;
std::stringstream message; std::stringstream message;

View File

@ -0,0 +1,2 @@
optimize_final 200001 1 1
optimize_final 202001 1 1

View File

@ -0,0 +1,18 @@
DROP TABLE IF EXISTS optimize_final;
CREATE TABLE optimize_final(t DateTime, x Int32) ENGINE = MergeTree() PARTITION BY toYYYYMM(t) ORDER BY x;
INSERT INTO optimize_final SELECT toDate('2000-01-01'), number FROM numbers(5);
INSERT INTO optimize_final SELECT toDate('2000-01-01'), number + 5 FROM numbers(5);
OPTIMIZE TABLE optimize_final FINAL;
INSERT INTO optimize_final SELECT toDate('2020-01-01'), number FROM numbers(5);
INSERT INTO optimize_final SELECT toDate('2020-01-01'), number + 5 FROM numbers(5);
OPTIMIZE TABLE optimize_final FINAL;
SELECT table, partition, active, level from system.parts where table = 'optimize_final' and active = 1;
DROP TABLE optimize_final;