From 8216a24abe25bcb1bd6865892c35e83bdd9c2be5 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 7 Nov 2023 16:56:06 +0100 Subject: [PATCH] move storage_metadata_write_full_object_key setting to the server scope --- src/Core/ServerSettings.h | 3 ++- src/Core/Settings.h | 1 - .../DiskObjectStorageMetadata.cpp | 4 ++-- .../configs/new_node.xml | 5 ++++ .../configs/old_node.xml | 5 ++++ .../configs/settings_new.xml | 11 --------- .../configs/switching_node.xml | 5 ++++ .../test_backward_compatibility.py | 24 +++++++++---------- 8 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 tests/integration/test_remote_blobs_naming/configs/new_node.xml create mode 100644 tests/integration/test_remote_blobs_naming/configs/old_node.xml delete mode 100644 tests/integration/test_remote_blobs_naming/configs/settings_new.xml create mode 100644 tests/integration/test_remote_blobs_naming/configs/switching_node.xml diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 523301a8933..34a6296fb55 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -98,7 +98,8 @@ namespace DB M(Double, total_memory_tracker_sample_probability, 0, "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ M(UInt64, total_memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ M(UInt64, total_memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ - M(Bool, validate_tcp_client_information, false, "Validate client_information in the query packet over the native TCP protocol.", 0) + M(Bool, validate_tcp_client_information, false, "Validate client_information in the query packet over the native TCP protocol.", 0) \ + M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \ DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index aa5c8569be6..2d42c3e185a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -288,7 +288,6 @@ class IColumn; M(UInt64, http_response_buffer_size, 0, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \ \ M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0) \ - M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \ \ M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ \ diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 9e501cc9342..e923e49a7f1 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -6,7 +6,7 @@ #include #include #include - +#include #include namespace DB @@ -204,7 +204,7 @@ void DiskObjectStorageMetadata::addObject(ObjectStorageKey key, size_t size) bool DiskObjectStorageMetadata::getWriteFullObjectKeySetting() { #ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD - return Context::getGlobalContextInstance()->getSettings().storage_metadata_write_full_object_key; + return Context::getGlobalContextInstance()->getServerSettings().storage_metadata_write_full_object_key; #else return false; #endif diff --git a/tests/integration/test_remote_blobs_naming/configs/new_node.xml b/tests/integration/test_remote_blobs_naming/configs/new_node.xml new file mode 100644 index 00000000000..2bb8d49ec4b --- /dev/null +++ b/tests/integration/test_remote_blobs_naming/configs/new_node.xml @@ -0,0 +1,5 @@ + + + + 1 + diff --git a/tests/integration/test_remote_blobs_naming/configs/old_node.xml b/tests/integration/test_remote_blobs_naming/configs/old_node.xml new file mode 100644 index 00000000000..6195d53a2af --- /dev/null +++ b/tests/integration/test_remote_blobs_naming/configs/old_node.xml @@ -0,0 +1,5 @@ + + + + 0 + diff --git a/tests/integration/test_remote_blobs_naming/configs/settings_new.xml b/tests/integration/test_remote_blobs_naming/configs/settings_new.xml deleted file mode 100644 index 1c5e0b0ee60..00000000000 --- a/tests/integration/test_remote_blobs_naming/configs/settings_new.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - 1 - 1 - 1 - - - diff --git a/tests/integration/test_remote_blobs_naming/configs/switching_node.xml b/tests/integration/test_remote_blobs_naming/configs/switching_node.xml new file mode 100644 index 00000000000..6195d53a2af --- /dev/null +++ b/tests/integration/test_remote_blobs_naming/configs/switching_node.xml @@ -0,0 +1,5 @@ + + + + 0 + diff --git a/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py b/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py index 22582f90ba6..485bf73dad1 100644 --- a/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py +++ b/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py @@ -13,6 +13,7 @@ def cluster(): cluster.add_instance( "node", main_configs=[ + "configs/old_node.xml", "configs/storage_conf.xml", ], user_configs=[ @@ -25,10 +26,11 @@ def cluster(): cluster.add_instance( "new_node", main_configs=[ + "configs/new_node.xml", "configs/storage_conf.xml", ], user_configs=[ - "configs/settings_new.xml", + "configs/settings.xml", ], with_minio=True, macros={"replica": "2"}, @@ -37,6 +39,7 @@ def cluster(): cluster.add_instance( "switching_node", main_configs=[ + "configs/switching_node.xml", "configs/storage_conf.xml", ], user_configs=[ @@ -238,24 +241,21 @@ def test_replicated_merge_tree(cluster, storage_policy): def switch_config_write_full_object_key(node, enable): - setting_path = "/etc/clickhouse-server/users.d/settings.xml" + setting_path = "/etc/clickhouse-server/config.d/switching_node.xml" + + is_on = "1<" + is_off = "0<" + data = read_file(node, setting_path) assert data != "" + assert is_on in data or is_off in data - is_on = "1" - is_off = "0" - - enable_line = is_off if enable: - enable_line = is_on - - if is_on in data: - data = data.replace(is_on, enable_line) + node.replace_in_config(setting_path, is_off, is_on) else: - data = data.replace(is_off, enable_line) + node.replace_in_config(setting_path, is_on, is_off) - write_file(node, setting_path, data) node.restart_clickhouse()