ClickHouse/src/Storages/MergeTree/MergeTreeSettings.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

162 lines
6.7 KiB
C++
Raw Normal View History

#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Poco/Util/AbstractConfiguration.h>
2023-02-04 18:41:41 +00:00
#include <Disks/getOrCreateDiskFromAST.h>
#include <Parsers/ASTCreateQuery.h>
2019-03-23 22:45:28 +00:00
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTFunction.h>
2023-02-04 18:31:19 +00:00
#include <Parsers/FieldFromAST.h>
#include <Core/Field.h>
#include <Common/Exception.h>
2023-02-04 18:31:19 +00:00
#include <Common/logger_useful.h>
#include <Core/Settings.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNKNOWN_SETTING;
2020-10-26 04:39:20 +00:00
extern const int BAD_ARGUMENTS;
}
IMPLEMENT_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS)
void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config)
{
if (!config.has(config_elem))
return;
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(config_elem, config_keys);
try
{
for (const String & key : config_keys)
set(key, config.getString(config_elem + "." + key));
}
catch (Exception & e)
{
if (e.code() == ErrorCodes::UNKNOWN_SETTING)
2020-10-26 03:35:57 +00:00
e.addMessage("in MergeTree config");
throw;
}
}
2023-02-04 18:31:19 +00:00
void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr context)
{
if (storage_def.settings)
{
try
{
2023-02-04 18:31:19 +00:00
auto changes = storage_def.settings->changes;
{
for (auto & [name, value] : changes)
{
CustomType custom;
if (value.tryGet<CustomType>(custom) && 0 == strcmp(custom.getTypeName(), "AST"))
{
auto ast = dynamic_cast<const FieldFromASTImpl &>(custom.getImpl()).ast;
if (ast && isDiskFunction(ast))
{
const auto & ast_function = assert_cast<const ASTFunction &>(*ast);
auto disk_name = getOrCreateDiskFromDiskAST(ast_function, context);
LOG_TRACE(&Poco::Logger::get("MergeTreeSettings"), "Created custom disk {}", disk_name);
value = disk_name;
break;
}
}
}
}
applyChanges(changes);
}
catch (Exception & e)
{
if (e.code() == ErrorCodes::UNKNOWN_SETTING)
2020-10-26 03:35:57 +00:00
e.addMessage("for storage " + storage_def.engine->name);
throw;
}
}
else
{
auto settings_ast = std::make_shared<ASTSetQuery>();
settings_ast->is_standalone = false;
storage_def.set(storage_def.settings, settings_ast);
}
SettingsChanges & changes = storage_def.settings->changes;
#define ADD_IF_ABSENT(NAME) \
if (std::find_if(changes.begin(), changes.end(), \
[](const SettingChange & c) { return c.name == #NAME; }) \
== changes.end()) \
2020-03-18 02:02:24 +00:00
changes.push_back(SettingChange{#NAME, (NAME).value});
APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT)
#undef ADD_IF_ABSENT
}
void MergeTreeSettings::sanityCheck(size_t background_pool_tasks) const
2020-07-30 19:08:13 +00:00
{
if (number_of_free_entries_in_pool_to_execute_mutation > background_pool_tasks)
2020-07-30 19:08:13 +00:00
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_execute_mutation' setting"
" ({}) (default values are defined in <merge_tree> section of config.xml"
" or the value can be specified per table in SETTINGS section of CREATE TABLE query)"
" is greater than the value of 'background_pool_size'*'background_merges_mutations_concurrency_ratio'"
2020-07-30 19:08:13 +00:00
" ({}) (the value is defined in users.xml for default profile)."
" This indicates incorrect configuration because mutations cannot work with these settings.",
number_of_free_entries_in_pool_to_execute_mutation,
background_pool_tasks);
2020-07-30 19:08:13 +00:00
}
if (number_of_free_entries_in_pool_to_lower_max_size_of_merge > background_pool_tasks)
2020-07-30 19:08:13 +00:00
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting"
" ({}) (default values are defined in <merge_tree> section of config.xml"
" or the value can be specified per table in SETTINGS section of CREATE TABLE query)"
" is greater than the value of 'background_pool_size'*'background_merges_mutations_concurrency_ratio'"
2020-07-30 19:08:13 +00:00
" ({}) (the value is defined in users.xml for default profile)."
" This indicates incorrect configuration because the maximum size of merge will be always lowered.",
2020-08-13 15:41:26 +00:00
number_of_free_entries_in_pool_to_lower_max_size_of_merge,
background_pool_tasks);
2020-07-30 19:08:13 +00:00
}
2022-12-25 15:06:34 +00:00
// Zero index_granularity is nonsensical.
if (index_granularity < 1)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"index_granularity: value {} makes no sense",
index_granularity);
}
2021-03-08 09:38:07 +00:00
// The min_index_granularity_bytes value is 1024 b and index_granularity_bytes is 10 mb by default.
// If index_granularity_bytes is not disabled i.e > 0 b, then always ensure that it's greater than
// min_index_granularity_bytes. This is mainly a safeguard against accidents whereby a really low
// index_granularity_bytes SETTING of 1b can create really large parts with large marks.
if (index_granularity_bytes > 0 && index_granularity_bytes < min_index_granularity_bytes)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"index_granularity_bytes: {} is lower than specified min_index_granularity_bytes: {}",
index_granularity_bytes,
min_index_granularity_bytes);
}
// If min_bytes_to_rebalance_partition_over_jbod is not disabled i.e > 0 b, then always ensure that
// it's not less than min_bytes_to_rebalance_partition_over_jbod. This is a safeguard to avoid tiny
// parts to participate JBOD balancer which will slow down the merge process.
if (min_bytes_to_rebalance_partition_over_jbod > 0
&& min_bytes_to_rebalance_partition_over_jbod < max_bytes_to_merge_at_max_space_in_pool / 1024)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
2022-03-10 10:29:42 +00:00
"min_bytes_to_rebalance_partition_over_jbod: {} is lower than specified max_bytes_to_merge_at_max_space_in_pool / 1024: {}",
2021-03-08 09:38:07 +00:00
min_bytes_to_rebalance_partition_over_jbod,
max_bytes_to_merge_at_max_space_in_pool / 1024);
}
}
}