From a4eeed50b64eaae9e84a89b70651996b37a99813 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 20 Sep 2023 12:49:15 +0000 Subject: [PATCH 1/3] Mark obsolete settings in system.merge_tree_settings as is_obsolete = 1 --- src/Storages/MergeTree/MergeTreeSettings.h | 49 +++++++++++++--------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 4f36da048c2..1c338f3817c 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -24,7 +24,7 @@ struct Settings; * and should not be changed by the user without a reason. */ -#define LIST_OF_MERGE_TREE_SETTINGS(M, ALIAS) \ +#define MERGE_TREE_SETTINGS(M, ALIAS) \ M(UInt64, min_compress_block_size, 0, "When granule is written, compress the data in buffer if the size of pending uncompressed data is larger or equal than the specified threshold. If this setting is not set, the corresponding global setting is used.", 0) \ M(UInt64, max_compress_block_size, 0, "Compress the pending uncompressed data in buffer if its size is larger or equal than the specified threshold. Block of data will be compressed even if the current granule is not finished. If this setting is not set, the corresponding global setting is used.", 0) \ M(UInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ @@ -165,6 +165,7 @@ struct Settings; M(Bool, allow_vertical_merges_from_compact_to_wide_parts, true, "Allows vertical merges from compact to wide parts. This settings must have the same value on all replicas", 0) \ M(Bool, enable_the_endpoint_id_with_zookeeper_name_prefix, false, "Enable the endpoint id with zookeeper name prefix for the replicated merge tree table", 0) \ M(UInt64, zero_copy_merge_mutation_min_parts_size_sleep_before_lock, 1ULL * 1024 * 1024 * 1024, "If zero copy replication is enabled sleep random amount of time before trying to lock depending on parts size for merge or mutation", 0) \ + M(Bool, allow_floating_point_partition_key, false, "Allow floating point as partition key", 0) \ \ /** Experimental/work in progress feature. Unsafe for production. */ \ M(UInt64, part_moves_between_shards_enable, 0, "Experimental/Incomplete feature to move parts between shards. Does not take into account sharding expressions.", 0) \ @@ -181,31 +182,39 @@ struct Settings; M(UInt64, marks_compress_block_size, 65536, "Mark compress block size, the actual size of the block to compress.", 0) \ M(UInt64, primary_key_compress_block_size, 65536, "Primary compress block size, the actual size of the block to compress.", 0) \ \ - /** Obsolete settings. Kept for backward compatibility only. */ \ - M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ - M(UInt64, check_delay_period, 60, "Obsolete setting, does nothing.", 0) \ - M(Bool, allow_floating_point_partition_key, false, "Allow floating point as partition key", 0) \ - M(UInt64, replicated_max_parallel_sends, 0, "Obsolete setting, does nothing.", 0) \ - M(UInt64, replicated_max_parallel_sends_for_table, 0, "Obsolete setting, does nothing.", 0) \ - M(UInt64, replicated_max_parallel_fetches, 0, "Obsolete setting, does nothing.", 0) \ - M(UInt64, replicated_max_parallel_fetches_for_table, 0, "Obsolete setting, does nothing.", 0) \ - M(Bool, write_final_mark, true, "Obsolete setting, does nothing.", 0) \ - M(UInt64, min_bytes_for_compact_part, 0, "Obsolete setting, does nothing.", 0) \ - M(UInt64, min_rows_for_compact_part, 0, "Obsolete setting, does nothing.", 0) \ - M(Bool, in_memory_parts_enable_wal, true, "Obsolete setting, does nothing.", 0) \ - M(UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024, "Obsolete setting, does nothing.", 0) \ - M(UInt64, write_ahead_log_bytes_to_fsync, 100ULL * 1024 * 1024, "Obsolete setting, does nothing.", 0) \ - M(UInt64, write_ahead_log_interval_ms_to_fsync, 100, "Obsolete setting, does nothing.", 0) \ - M(Bool, in_memory_parts_insert_sync, false, "Obsolete setting, does nothing.", 0) \ - M(MaxThreads, max_part_loading_threads, 0, "Obsolete setting, does nothing.", 0) \ - M(MaxThreads, max_part_removal_threads, 0, "Obsolete setting, does nothing.", 0) \ - M(Bool, use_metadata_cache, false, "Obsolete setting, does nothing.", 0) \ + +#define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \ + M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) + +#define OBSOLETE_MERGE_TREE_SETTINGS(M, ALIAS) \ + /** Obsolete settings that do nothing but left for compatibility reasons. */ \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, min_relative_delay_to_yield_leadership, 120) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, check_delay_period, 60) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, replicated_max_parallel_sends, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, replicated_max_parallel_sends_for_table, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, replicated_max_parallel_fetches, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, replicated_max_parallel_fetches_for_table, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, Bool, write_final_mark, true) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, min_bytes_for_compact_part, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, min_rows_for_compact_part, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, Bool, in_memory_parts_enable_wal, true) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, write_ahead_log_max_bytes, 1024 * 1024 * 1024) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, write_ahead_log_bytes_to_fsync, 100ULL * 1024 * 1024) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, UInt64, write_ahead_log_interval_ms_to_fsync, 100) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, Bool, in_memory_parts_insert_sync, false) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, MaxThreads, max_part_loading_threads, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, MaxThreads, max_part_removal_threads, 0) \ + MAKE_OBSOLETE_MERGE_TREE_SETTING(M, Bool, use_metadata_cache, false) \ /// Settings that should not change after the creation of a table. /// NOLINTNEXTLINE #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \ M(index_granularity) +#define LIST_OF_MERGE_TREE_SETTINGS(M, ALIAS) \ + MERGE_TREE_SETTINGS(M, ALIAS) \ + OBSOLETE_MERGE_TREE_SETTINGS(M, ALIAS) + DECLARE_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS) From e954ffe939f420bd45ba4be1233bf93437096d1b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 26 Sep 2023 09:51:48 +0000 Subject: [PATCH 2/3] Cleanups --- src/Core/Settings.h | 8 ++++---- src/Storages/MergeTree/MergeTreeSettings.h | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 94968735800..19182db16fb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -800,9 +800,9 @@ class IColumn; M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ M(UInt64, extract_kvp_max_pairs_per_row, 1000, "Max number pairs that can be produced by extractKeyValuePairs function. Used to safeguard against consuming too much memory.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ - M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0)\ - M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \ - M(Bool, ignore_access_denied_multidirectory_globs, false, "Ignore access denied errors when processing multi-directory globs for file & HDFS.", 0)\ + M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0) \ + M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \ + M(Bool, ignore_access_denied_multidirectory_globs, false, "Ignore access denied errors when processing multi-directory globs for file & HDFS.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. @@ -851,7 +851,7 @@ class IColumn; MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_sends_network_bandwidth_for_server, 0) \ /* ---- */ \ MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \ - MAKE_OBSOLETE(M, UInt64, max_pipeline_depth, 0) \ + MAKE_OBSOLETE(M, UInt64, max_pipeline_depth, 0) \ MAKE_OBSOLETE(M, Seconds, temporary_live_view_timeout, 1) \ MAKE_OBSOLETE(M, Milliseconds, async_insert_cleanup_timeout_ms, 1000) \ MAKE_OBSOLETE(M, Bool, optimize_fuse_sum_count_avg, 0) \ diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 1c338f3817c..79df52f2aa9 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -181,7 +181,6 @@ struct Settings; M(String, primary_key_compression_codec, "ZSTD(3)", "Compression encoding used by primary, primary key is small enough and cached, so the default compression is ZSTD(3).", 0) \ M(UInt64, marks_compress_block_size, 65536, "Mark compress block size, the actual size of the block to compress.", 0) \ M(UInt64, primary_key_compress_block_size, 65536, "Primary compress block size, the actual size of the block to compress.", 0) \ - \ #define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \ M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) From 978a0c9cab4a36bbe95e1d75270040e10083805a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 26 Sep 2023 10:45:50 +0000 Subject: [PATCH 3/3] + test --- .../system-tables/merge_tree_settings.md | 2 +- .../system-tables/server_settings.md | 2 +- docs/en/operations/system-tables/settings.md | 2 +- .../02888_obsolete_settings.reference | 63 +++++++++++++++++++ .../0_stateless/02888_obsolete_settings.sql | 8 +++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02888_obsolete_settings.reference create mode 100644 tests/queries/0_stateless/02888_obsolete_settings.sql diff --git a/docs/en/operations/system-tables/merge_tree_settings.md b/docs/en/operations/system-tables/merge_tree_settings.md index 557835ce3b6..48217d63f9d 100644 --- a/docs/en/operations/system-tables/merge_tree_settings.md +++ b/docs/en/operations/system-tables/merge_tree_settings.md @@ -17,7 +17,7 @@ Columns: - `0` — Current user can change the setting. - `1` — Current user can’t change the setting. - `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value). -- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete. +- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete. **Example** ```sql diff --git a/docs/en/operations/system-tables/server_settings.md b/docs/en/operations/system-tables/server_settings.md index df482261ae8..7efe605ccef 100644 --- a/docs/en/operations/system-tables/server_settings.md +++ b/docs/en/operations/system-tables/server_settings.md @@ -14,7 +14,7 @@ Columns: - `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting was specified in `config.xml` - `description` ([String](../../sql-reference/data-types/string.md)) — Short server setting description. - `type` ([String](../../sql-reference/data-types/string.md)) — Server setting value type. -- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete. +- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete. **Example** diff --git a/docs/en/operations/system-tables/settings.md b/docs/en/operations/system-tables/settings.md index 7dd2345a2d0..a04e095e990 100644 --- a/docs/en/operations/system-tables/settings.md +++ b/docs/en/operations/system-tables/settings.md @@ -17,7 +17,7 @@ Columns: - `0` — Current user can change the setting. - `1` — Current user can’t change the setting. - `default` ([String](../../sql-reference/data-types/string.md)) — Setting default value. -- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete. +- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete. **Example** diff --git a/tests/queries/0_stateless/02888_obsolete_settings.reference b/tests/queries/0_stateless/02888_obsolete_settings.reference new file mode 100644 index 00000000000..af4e486ffb5 --- /dev/null +++ b/tests/queries/0_stateless/02888_obsolete_settings.reference @@ -0,0 +1,63 @@ +-- Obsolete server settings +-- Obsolete general settings +allow_experimental_bigint_types +allow_experimental_database_atomic +allow_experimental_geo_types +allow_experimental_map_type +allow_experimental_window_functions +async_insert_cleanup_timeout_ms +async_insert_stale_timeout_ms +async_insert_threads +background_buffer_flush_schedule_pool_size +background_common_pool_size +background_distributed_schedule_pool_size +background_fetches_pool_size +background_merges_mutations_concurrency_ratio +background_message_broker_schedule_pool_size +background_move_pool_size +background_pool_size +background_schedule_pool_size +backup_threads +database_replicated_ddl_output +default_database_engine +drain_timeout +enable_debug_queries +handle_kafka_error_mode +input_format_arrow_import_nested +input_format_orc_import_nested +input_format_parquet_import_nested +max_alter_threads +max_memory_usage_for_all_queries +max_pipeline_depth +max_remote_read_network_bandwidth_for_server +max_remote_write_network_bandwidth_for_server +max_replicated_fetches_network_bandwidth_for_server +max_replicated_sends_network_bandwidth_for_server +merge_tree_clear_old_parts_interval_seconds +merge_tree_clear_old_temporary_directories_interval_seconds +multiple_joins_rewriter_version +odbc_max_field_size +optimize_duplicate_order_by_and_distinct +optimize_fuse_sum_count_avg +partial_merge_join_optimizations +replication_alter_columns_timeout +restore_threads +temporary_live_view_timeout +-- Obsolete merge tree settings +check_delay_period +in_memory_parts_enable_wal +in_memory_parts_insert_sync +max_part_loading_threads +max_part_removal_threads +min_bytes_for_compact_part +min_relative_delay_to_yield_leadership +min_rows_for_compact_part +replicated_max_parallel_fetches +replicated_max_parallel_fetches_for_table +replicated_max_parallel_sends +replicated_max_parallel_sends_for_table +use_metadata_cache +write_ahead_log_bytes_to_fsync +write_ahead_log_interval_ms_to_fsync +write_ahead_log_max_bytes +write_final_mark diff --git a/tests/queries/0_stateless/02888_obsolete_settings.sql b/tests/queries/0_stateless/02888_obsolete_settings.sql new file mode 100644 index 00000000000..4d336997ad2 --- /dev/null +++ b/tests/queries/0_stateless/02888_obsolete_settings.sql @@ -0,0 +1,8 @@ +SELECT '-- Obsolete server settings'; +SELECT name FROM system.server_settings WHERE is_obsolete = 1 ORDER BY name; + +SELECT '-- Obsolete general settings'; +SELECT name FROM system.settings WHERE is_obsolete = 1 ORDER BY name; + +SELECT '-- Obsolete merge tree settings'; +SELECT name FROM system.merge_tree_settings WHERE is_obsolete = 1 ORDER BY name;