#pragma once #include #include #include namespace DB { namespace ErrorCodes { extern const int INVALID_CONFIG_PARAMETER; } /** Advanced settings of MergeTree. * Could be loaded from config. */ struct MergeTreeSettings { /** Merge settings. */ /// Maximum in total size of parts to merge, when there are maximum (minimum) free threads in background pool (or entries in replication queue). size_t max_bytes_to_merge_at_max_space_in_pool = 100ULL * 1024 * 1024 * 1024; size_t max_bytes_to_merge_at_min_space_in_pool = 1024 * 1024; /// How many tasks of merging parts are allowed simultaneously in ReplicatedMergeTree queue. size_t max_replicated_merges_in_queue = 16; /// When there is less than specified number of free entries in pool (or replicated queue), /// start to lower maximum size of merge to process (or to put in queue). /// This is to allow small merges to process - not filling the pool with long running merges. size_t number_of_free_entries_in_pool_to_lower_max_size_of_merge = 8; /// How many seconds to keep obsolete parts. time_t old_parts_lifetime = 8 * 60; /// How many seconds to keep tmp_-directories. time_t temporary_directories_lifetime = 86400; /** Inserts settings. */ /// If table contains at least that many active parts, artificially slow down insert into table. size_t parts_to_delay_insert = 150; /// If more than this number active parts, throw 'Too much parts ...' exception size_t parts_to_throw_insert = 300; /// Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts. size_t max_delay_to_insert = 200; /** Replication settings. */ /// How many last blocks of hashes should be kept in ZooKeeper. size_t replicated_deduplication_window = 100; /// Keep about this number of last records in ZooKeeper log, even if they are obsolete. /// It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning. size_t replicated_logs_to_keep = 100; /// After specified amount of time passed after replication log entry creation /// and sum size of parts is greater than threshold, /// prefer fetching merged part from replica instead of doing merge locally. /// To speed up very long merges. time_t prefer_fetch_merged_part_time_threshold = 3600; size_t prefer_fetch_merged_part_size_threshold = 10ULL * 1024 * 1024 * 1024; /// Max broken parts, if more - deny automatic deletion. size_t max_suspicious_broken_parts = 10; /// Not apply ALTER if number of files for modification(deletion, addition) more than this. size_t max_files_to_modify_in_alter_columns = 75; /// Not apply ALTER, if number of files for deletion more than this. size_t max_files_to_remove_in_alter_columns = 50; /// Maximum number of errors during parts loading, while ReplicatedMergeTree still allowed to start. size_t replicated_max_unexpected_parts = 3; size_t replicated_max_unexpectedly_merged_parts = 2; size_t replicated_max_missing_obsolete_parts = 5; size_t replicated_max_missing_active_parts = 20; /// Limit parallel fetches size_t replicated_max_parallel_fetches = 4; size_t replicated_max_parallel_fetches_for_table = 2; /// Limit parallel sends size_t replicated_max_parallel_sends = 4; size_t replicated_max_parallel_sends_for_table = 2; /// If ration of wrong parts to total number of parts is less than this - allow to start anyway. double replicated_max_ratio_of_wrong_parts = 0.05; /// In seconds. size_t zookeeper_session_expiration_check_period = 60; /** Check delay of replicas settings. */ /// Period to check replication delay and compare with other replicas. size_t check_delay_period = 60; /// Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited. size_t min_relative_delay_to_yield_leadership = 120; /// Minimal delay from other replicas to close, stop serving requests and not return Ok during status check. size_t min_relative_delay_to_close = 300; /// Minimal absolute delay to close, stop serving requests and not return Ok during status check. size_t min_absolute_delay_to_close = 0; /// Enable usage of Vertical merge algorithm. size_t enable_vertical_merge_algorithm = 1; /// Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm size_t vertical_merge_algorithm_min_rows_to_activate = 16 * DEFAULT_MERGE_BLOCK_SIZE; /// Minimal amount of non-PK columns to activate Vertical merge algorithm size_t vertical_merge_algorithm_min_columns_to_activate = 11; void loadFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config) { #define SET(NAME, GETTER) \ try \ { \ NAME = config.GETTER(config_elem + "." #NAME, NAME); \ } \ catch (const Poco::Exception & e) \ { \ throw Exception( \ "Invalid config parameter: " + config_elem + "/" #NAME + ": " + e.message() + ".", \ ErrorCodes::INVALID_CONFIG_PARAMETER); \ } SET(max_bytes_to_merge_at_max_space_in_pool, getUInt64); SET(max_bytes_to_merge_at_min_space_in_pool, getUInt64); SET(max_replicated_merges_in_queue, getUInt64); SET(old_parts_lifetime, getUInt64); SET(temporary_directories_lifetime, getUInt64); SET(parts_to_delay_insert, getUInt64); SET(parts_to_throw_insert, getUInt64); SET(max_delay_to_insert, getUInt64); SET(replicated_deduplication_window, getUInt64); SET(replicated_logs_to_keep, getUInt64); SET(prefer_fetch_merged_part_time_threshold, getUInt64); SET(prefer_fetch_merged_part_size_threshold, getUInt64); SET(max_suspicious_broken_parts, getUInt64); SET(max_files_to_modify_in_alter_columns, getUInt64); SET(max_files_to_remove_in_alter_columns, getUInt64); SET(replicated_max_unexpected_parts, getUInt64); SET(replicated_max_unexpectedly_merged_parts, getUInt64); SET(replicated_max_missing_obsolete_parts, getUInt64); SET(replicated_max_missing_active_parts, getUInt64); SET(replicated_max_parallel_fetches, getUInt64); SET(replicated_max_parallel_fetches_for_table, getUInt64); SET(replicated_max_parallel_sends, getUInt64); SET(replicated_max_parallel_sends_for_table, getUInt64); SET(replicated_max_ratio_of_wrong_parts, getDouble); SET(zookeeper_session_expiration_check_period, getUInt64); SET(check_delay_period, getUInt64); SET(min_relative_delay_to_yield_leadership, getUInt64); SET(min_relative_delay_to_close, getUInt64); SET(min_absolute_delay_to_close, getUInt64); SET(enable_vertical_merge_algorithm, getUInt64); SET(vertical_merge_algorithm_min_rows_to_activate, getUInt64); SET(vertical_merge_algorithm_min_columns_to_activate, getUInt64); #undef SET } }; }