thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Corrected implementation for check of new settings and fix lint of settings change history.

Browse Source
This commit is contained in:
marco-vb 2024-09-13 13:21:35 +00:00
parent 8299b31d47
commit ddc506a677
2 changed files with 29 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Core/SettingsChangesHistory.cpp
View File

@ -82,9 +82,9 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"},
{"join_to_sort_minimum_perkey_rows", 0, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys"},
{"join_to_sort_maximum_table_rows", 0, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join"},
{"allow_experimental_join_right_table_sorting", false, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join"}
{"allow_experimental_join_right_table_sorting", false, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join"},
{"min_free_disk_bytes_to_throw_insert", 0, 0, "Maintain some free disk space bytes from inserts while still allowing for temporary writing."},
{"min_free_disk_ratio_to_throw_insert", 0.0, 0.0, "Maintain some free disk space bytes expressed as ratio to total disk space from inserts while still allowing for temporary writing."},
{"min_free_disk_ratio_to_throw_insert", 0.0, 0.0, "Maintain some free disk space bytes expressed as ratio to total disk space from inserts while still allowing for temporary writing."},
}
},
{"24.8",

48
src/Storages/MergeTree/MergeTreeDataWriter.cpp
View File

@ -14,7 +14,6 @@
#include <Storages/MergeTree/MergedBlockOutputStream.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Storages/MergeTree/RowOrderOptimizer.h>
#include <Common/CurrentThread.h>
#include <Common/ElapsedTimeProfileEventIncrement.h>
#include <Common/Exception.h>
#include <Common/HashTable/HashMap.h>
@ -61,6 +60,7 @@ namespace ErrorCodes
extern const int ABORTED;
extern const int LOGICAL_ERROR;
extern const int TOO_MANY_PARTS;
extern const int NOT_ENOUGH_SPACE;
}
namespace
@ -554,6 +554,31 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
VolumePtr volume = data.getStoragePolicy()->getVolume(0);
VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume);
const auto & data_settings = data.getSettings();
{
const UInt64 min_bytes = data_settings->min_free_disk_bytes_to_throw_insert;
const Float64 min_ratio = data_settings->min_free_disk_ratio_to_throw_insert;
const auto disk = data_part_volume->getDisk();
const UInt64 total_disk_bytes = *disk->getTotalSpace();
const UInt64 free_disk_bytes = *disk->getUnreservedSpace();
const UInt64 min_bytes_from_ratio = static_cast<UInt64>(min_ratio * total_disk_bytes);
const UInt64 needed_free_bytes = std::max(min_bytes, min_bytes_from_ratio);
if (needed_free_bytes > free_disk_bytes)
{
throw Exception(
ErrorCodes::NOT_ENOUGH_SPACE,
"Could not perform insert: less than {} free bytes in disk space. "
"Configure this limit with user settings {} or {}",
needed_free_bytes,
"min_free_disk_bytes_to_throw_insert",
"min_free_disk_ratio_to_throw_insert");
}
}
auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir)
.withPartFormat(data.choosePartFormat(expected_size, block.rows()))
.withPartInfo(new_part_info)
@ -565,8 +590,6 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
if (data.storage_settings.get()->assign_part_uuids)
new_data_part->uuid = UUIDHelpers::generateV4();
const auto & data_settings = data.getSettings();
SerializationInfo::Settings settings{data_settings->ratio_of_defaults_for_sparse_serialization, true};
SerializationInfoByName infos(columns, settings);
infos.add(block);
@ -690,25 +713,8 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl(
/// Size of part would not be greater than block.bytes() + epsilon
size_t expected_size = block.bytes();
// If not temporary insert try to reserve respecting min free disk bytes
size_t reserve_extra = 0;
if (!is_temp)
{
const auto context = CurrentThread::getQueryContext();
const auto * settings = context ? &context->getSettingsRef() : nullptr;
const UInt64 min_bytes = settings->min_free_disk_bytes_to_throw_insert;
const Float64 min_ratio = settings->min_free_disk_ratio_to_throw_insert;
const auto total_disk_space = parent_part->getDataPartStorage().calculateTotalSizeOnDisk();
const UInt64 min_bytes_from_ratio = static_cast<UInt64>(min_ratio * total_disk_space);
reserve_extra = std::min(min_bytes, min_bytes_from_ratio);
}
// just check if there is enough space on parent volume
// down the line in reserving space there is concurrency safety so no need to worry about 'over-reserving'
MergeTreeData::reserveSpace(expected_size + reserve_extra, parent_part->getDataPartStorage());
MergeTreeData::reserveSpace(expected_size, parent_part->getDataPartStorage());
part_type = data.choosePartFormatOnDisk(expected_size, block.rows()).part_type;
auto new_data_part = parent_part->getProjectionPartBuilder(part_name, is_temp).withPartType(part_type).build();