Fix bug and add more trash to test

This commit is contained in:
alesapin 2021-07-06 19:51:23 +03:00
parent e9540f0621
commit a8fdc41193
5 changed files with 43 additions and 7 deletions

View File

@ -46,7 +46,9 @@ void DropPartsRanges::removeDropRange(const ReplicatedMergeTreeLogEntryPtr & ent
if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove entry of type {} from drop ranges, expected DROP_RANGE", entry->typeToString()); throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove entry of type {} from drop ranges, expected DROP_RANGE", entry->typeToString());
drop_ranges.erase(entry->znode_name); auto it = drop_ranges.find(entry->znode_name);
assert(it != drop_ranges.end());
drop_ranges.erase(it);
} }
bool DropPartsRanges::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const bool DropPartsRanges::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const

View File

@ -431,6 +431,16 @@ std::optional<String> ReplicatedMergeTreeLogEntryData::getDropRange(MergeTreeDat
return {}; return {};
} }
bool ReplicatedMergeTreeLogEntryData::isDropPart(MergeTreeDataFormatVersion format_version) const
{
if (type == DROP_RANGE)
{
auto drop_range_info = MergeTreePartInfo::fromPartName(new_part_name, format_version);
return !drop_range_info.isFakeDropRangePart();
}
return false;
}
Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormatVersion format_version) const Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormatVersion format_version) const
{ {
/// Doesn't produce any part /// Doesn't produce any part

View File

@ -143,6 +143,10 @@ struct ReplicatedMergeTreeLogEntryData
/// Returns fake part for drop range (for DROP_RANGE and REPLACE_RANGE) /// Returns fake part for drop range (for DROP_RANGE and REPLACE_RANGE)
std::optional<String> getDropRange(MergeTreeDataFormatVersion format_version) const; std::optional<String> getDropRange(MergeTreeDataFormatVersion format_version) const;
/// This entry is DROP PART, not DROP PARTITION. They both have same
/// DROP_RANGE entry type, but differs in information about drop range.
bool isDropPart(MergeTreeDataFormatVersion format_version) const;
/// Access under queue_mutex, see ReplicatedMergeTreeQueue. /// Access under queue_mutex, see ReplicatedMergeTreeQueue.
bool currently_executing = false; /// Whether the action is executing now. bool currently_executing = false; /// Whether the action is executing now.
bool removed_by_other_entry = false; bool removed_by_other_entry = false;

View File

@ -170,12 +170,11 @@ void ReplicatedMergeTreeQueue::insertUnlocked(
else else
{ {
drop_ranges.addDropRange(entry); drop_ranges.addDropRange(entry);
auto drop_range = *entry->getDropRange(format_version);
/// DROP PARTS (not DROP PARTITIONS) removes parts from virtual parts. /// DROP PART remove parts, so we remove it from virtual parts to
MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(drop_range, format_version); /// preserve invariant virtual_parts = current_parts + queue
if (!drop_range_info.isFakeDropRangePart() && virtual_parts.getContainingPart(drop_range_info) == drop_range) if (entry->isDropPart(format_version))
virtual_parts.removePartAndCoveredParts(drop_range); virtual_parts.removePartAndCoveredParts(*entry->getDropRange(format_version));
queue.push_front(entry); queue.push_front(entry);
} }
@ -266,7 +265,15 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval(
if (auto drop_range_part_name = entry->getDropRange(format_version)) if (auto drop_range_part_name = entry->getDropRange(format_version))
{ {
current_parts.remove(*drop_range_part_name); MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(*drop_range_part_name, format_version);
/// DROP PART doesn't have virtual parts so remove from current
/// parts all covered parts.
if (entry->isDropPart(format_version))
current_parts.removePartAndCoveredParts(*drop_range_part_name);
else
current_parts.remove(*drop_range_part_name);
virtual_parts.remove(*drop_range_part_name); virtual_parts.remove(*drop_range_part_name);
} }

View File

@ -72,6 +72,7 @@ function drop_partition_thread()
done done
} }
function optimize_thread() function optimize_thread()
{ {
while true; do while true; do
@ -85,12 +86,23 @@ function optimize_thread()
done done
} }
function drop_part_thread()
{
while true; do
REPLICA=$(($RANDOM % 16))
part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1")
$CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null
sleep 0.$RANDOM;
done
}
#export -f create_drop_thread; #export -f create_drop_thread;
export -f insert_thread; export -f insert_thread;
export -f move_partition_src_dst_thread; export -f move_partition_src_dst_thread;
export -f replace_partition_src_src_thread; export -f replace_partition_src_src_thread;
export -f drop_partition_thread; export -f drop_partition_thread;
export -f optimize_thread; export -f optimize_thread;
export -f drop_part_thread;
TIMEOUT=60 TIMEOUT=60
@ -102,6 +114,7 @@ timeout $TIMEOUT bash -c move_partition_src_dst_thread &
timeout $TIMEOUT bash -c replace_partition_src_src_thread & timeout $TIMEOUT bash -c replace_partition_src_src_thread &
timeout $TIMEOUT bash -c drop_partition_thread & timeout $TIMEOUT bash -c drop_partition_thread &
timeout $TIMEOUT bash -c optimize_thread & timeout $TIMEOUT bash -c optimize_thread &
timeout $TIMEOUT bash -c drop_part_thread &
wait wait
for ((i=0; i<16; i++)) do for ((i=0; i<16; i++)) do