thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add optimize_skip_merged_partitions

Browse Source
This commit is contained in:
Pavel Kruglov 2020-10-13 17:51:08 +03:00
parent f64344f7c5
commit 6d9b310f61
5 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
View File

@ -358,6 +358,7 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition(
const AllowedMergingPredicate & can_merge,
const String & partition_id,
bool final,
bool * is_single_merged_part,
String * out_disable_reason)
{
MergeTreeData::DataPartsVector parts = selectAllPartsFromPartition(partition_id);
@ -372,6 +373,12 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition(
return false;
}
if (final && data.getSettings()->optimize_skip_merged_partitions && parts.size() == 1 && parts[0]->info.level > 0)
{
*is_single_merged_part = true;
return false;
}
auto it = parts.begin();
auto prev_it = it;

1
src/Storages/MergeTree/MergeTreeDataMergerMutator.h
View File

@ -96,6 +96,7 @@ public:
const AllowedMergingPredicate & can_merge,
const String & partition_id,
bool final,
bool * is_single_merged_part,
String * out_disable_reason = nullptr);
/** Merge the parts.

1
src/Storages/MergeTree/MergeTreeSettings.h
View File

@ -48,6 +48,7 @@ struct Settings;
M(UInt64, write_ahead_log_bytes_to_fsync, 100ULL * 1024 * 1024, "Amount of bytes, accumulated in WAL to do fsync.", 0) \
M(UInt64, write_ahead_log_interval_ms_to_fsync, 100, "Interval in milliseconds after which fsync for WAL is being done.", 0) \
M(Bool, in_memory_parts_insert_sync, false, "If true insert of part with in-memory format will wait for fsync of WAL", 0) \
M(Bool, optimize_skip_merged_partitions, true, "Skip partitions with one part with level > 0 in optimize final", 0) \
\
/** Inserts settings. */ \
M(UInt64, parts_to_delay_insert, 150, "If table contains at least that many active parts in single partition, artificially slow down insert into table.", 0) \

7
src/Storages/StorageMergeTree.cpp
View File

@ -654,6 +654,7 @@ bool StorageMergeTree::merge(
};
bool selected = false;
bool is_single_merged_part = false;
if (partition_id.empty())
{
@ -682,7 +683,7 @@ bool StorageMergeTree::merge(
{
UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace();
selected = merger_mutator.selectAllPartsToMergeWithinPartition(
future_part, disk_space, can_merge, partition_id, final, out_disable_reason);
future_part, disk_space, can_merge, partition_id, final, &is_single_merged_part, out_disable_reason);
/// If final - we will wait for currently processing merges to finish and continue.
/// TODO Respect query settings for timeout
@ -709,6 +710,10 @@ bool StorageMergeTree::merge(
if (!selected)
{
if (final && is_single_merged_part)
{
return true;
}
if (out_disable_reason)
{
if (!out_disable_reason->empty())

13
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -3761,11 +3761,16 @@ bool StorageReplicatedMergeTree::optimize(
ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper);
FutureMergedMutatedPart future_merged_part;
bool is_single_merged_part = false;
bool selected = merger_mutator.selectAllPartsToMergeWithinPartition(
future_merged_part, disk_space, can_merge, partition_id, true, nullptr);
future_merged_part, disk_space, can_merge, partition_id, true, &is_single_merged_part, nullptr);
if (!selected)
{
if (is_single_merged_part)
return true;
break;
}
ReplicatedMergeTreeLogEntryData merge_entry;
CreateMergeEntryResult create_result = createLogEntryToMergeParts(
@ -3798,6 +3803,7 @@ bool StorageReplicatedMergeTree::optimize(
FutureMergedMutatedPart future_merged_part;
String disable_reason;
bool selected = false;
bool is_single_merged_part = false;
if (!partition)
{
selected = merger_mutator.selectPartsToMerge(
@ -3805,15 +3811,16 @@ bool StorageReplicatedMergeTree::optimize(
}
else
{
UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace();
String partition_id = getPartitionIDFromQuery(partition, query_context);
selected = merger_mutator.selectAllPartsToMergeWithinPartition(
future_merged_part, disk_space, can_merge, partition_id, final, &disable_reason);
future_merged_part, disk_space, can_merge, partition_id, final, &is_single_merged_part, &disable_reason);
}
if (!selected)
{
if (final && is_single_merged_part)
return true;
std::stringstream message;
message << "Cannot select parts for optimization";
if (!disable_reason.empty())