thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix creation of empty parts

Browse Source
This commit is contained in:
Alexander Tokmakov 2023-08-14 23:29:28 +02:00
parent 37d8155773
commit 376202f739
4 changed files with 18 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -8435,7 +8435,7 @@ void MergeTreeData::incrementMergedPartsProfileEvent(MergeTreeDataPartType type)
}
}
MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> MergeTreeData::createEmptyPart(
MergeTreePartInfo & new_part_info, const MergeTreePartition & partition, const String & new_part_name,
const MergeTreeTransactionPtr & txn)
{
@ -8454,6 +8454,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
ReservationPtr reservation = reserveSpacePreferringTTLRules(metadata_snapshot, 0, move_ttl_infos, time(nullptr), 0, true);
VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume);
auto tmp_dir_holder = getTemporaryPartDirectoryHolder(EMPTY_PART_TMP_PREFIX + new_part_name);
auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name)
.withBytesAndRowsOnDisk(0, 0)
.withPartInfo(new_part_info)
@ -8513,7 +8514,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart(
out.finalizePart(new_data_part, sync_on_insert);
new_data_part_storage->precommitTransaction();
return new_data_part;
return std::make_pair(std::move(new_data_part), std::move(tmp_dir_holder));
}
bool MergeTreeData::allowRemoveStaleMovingParts() const

4
src/Storages/MergeTree/MergeTreeData.h
View File

@ -936,7 +936,9 @@ public:
WriteAheadLogPtr getWriteAheadLog();
constexpr static auto EMPTY_PART_TMP_PREFIX = "tmp_empty_";
MergeTreeData::MutableDataPartPtr createEmptyPart(MergeTreePartInfo & new_part_info, const MergeTreePartition & partition, const String & new_part_name, const MergeTreeTransactionPtr & txn);
std::pair<MergeTreeData::MutableDataPartPtr, scope_guard> createEmptyPart(
MergeTreePartInfo & new_part_info, const MergeTreePartition & partition,
const String & new_part_name, const MergeTreeTransactionPtr & txn);
MergeTreeDataFormatVersion format_version;

31
src/Storages/StorageMergeTree.cpp
View File

@ -1653,11 +1653,7 @@ struct FutureNewEmptyPart
MergeTreePartition partition;
std::string part_name;
scope_guard tmp_dir_guard;
StorageMergeTree::MutableDataPartPtr data_part;
std::string getDirName() const { return StorageMergeTree::EMPTY_PART_TMP_PREFIX + part_name; }
};
using FutureNewEmptyParts = std::vector<FutureNewEmptyPart>;
@ -1688,19 +1684,19 @@ FutureNewEmptyParts initCoverageWithNewEmptyParts(const DataPartsVector & old_pa
return future_parts;
}
StorageMergeTree::MutableDataPartsVector createEmptyDataParts(MergeTreeData & data, FutureNewEmptyParts & future_parts, const MergeTreeTransactionPtr & txn)
std::pair<StorageMergeTree::MutableDataPartsVector, std::vector<scope_guard>> createEmptyDataParts(
MergeTreeData & data, FutureNewEmptyParts & future_parts, const MergeTreeTransactionPtr & txn)
{
StorageMergeTree::MutableDataPartsVector data_parts;
std::pair<StorageMergeTree::MutableDataPartsVector, std::vector<scope_guard>> data_parts;
for (auto & part: future_parts)
data_parts.push_back(data.createEmptyPart(part.part_info, part.partition, part.part_name, txn));
{
auto [new_data_part, tmp_dir_holder] = data.createEmptyPart(part.part_info, part.partition, part.part_name, txn);
data_parts.first.emplace_back(std::move(new_data_part));
data_parts.second.emplace_back(std::move(tmp_dir_holder));
}
return data_parts;
}
void captureTmpDirectoryHolders(MergeTreeData & data, FutureNewEmptyParts & future_parts)
{
for (auto & part : future_parts)
part.tmp_dir_guard = data.getTemporaryPartDirectoryHolder(part.getDirName());
}
void StorageMergeTree::renameAndCommitEmptyParts(MutableDataPartsVector & new_parts, Transaction & transaction)
{
@ -1767,9 +1763,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont
fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames(parts), ", "),
transaction.getTID());
captureTmpDirectoryHolders(*this, future_parts);
auto new_data_parts = createEmptyDataParts(*this, future_parts, txn);
auto [new_data_parts, tmp_dir_holders] = createEmptyDataParts(*this, future_parts, txn);
renameAndCommitEmptyParts(new_data_parts, transaction);
PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot()));
@ -1828,9 +1822,7 @@ void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPt
fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames({part}), ", "),
transaction.getTID());
captureTmpDirectoryHolders(*this, future_parts);
auto new_data_parts = createEmptyDataParts(*this, future_parts, txn);
auto [new_data_parts, tmp_dir_holders] = createEmptyDataParts(*this, future_parts, txn);
renameAndCommitEmptyParts(new_data_parts, transaction);
PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot()));
@ -1914,9 +1906,8 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, Cont
fmt::join(getPartsNames(future_parts), ", "), fmt::join(getPartsNames(parts), ", "),
transaction.getTID());
captureTmpDirectoryHolders(*this, future_parts);
auto new_data_parts = createEmptyDataParts(*this, future_parts, txn);
auto [new_data_parts, tmp_dir_holders] = createEmptyDataParts(*this, future_parts, txn);
renameAndCommitEmptyParts(new_data_parts, transaction);
PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot()));

2
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -9509,7 +9509,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP
}
}
MergeTreeData::MutableDataPartPtr new_data_part = createEmptyPart(new_part_info, partition, lost_part_name, NO_TRANSACTION_PTR);
auto [new_data_part, tmp_dir_holder] = createEmptyPart(new_part_info, partition, lost_part_name, NO_TRANSACTION_PTR);
new_data_part->setName(lost_part_name);
try