Merge pull request #56421 from CheSema/move-to-server-settings

move storage_metadata_write_full_object_key setting to the server scope

src/Core/ServerSettings.h

@@ -98,7 +98,8 @@ namespace DB
     M(Double, total_memory_tracker_sample_probability, 0, "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     M(UInt64, total_memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \
     M(UInt64, total_memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \
-    M(Bool, validate_tcp_client_information, false, "Validate client_information in the query packet over the native TCP protocol.", 0)
+    M(Bool, validate_tcp_client_information, false, "Validate client_information in the query packet over the native TCP protocol.", 0) \
+    M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \
 
 DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS)
 

src/Core/Settings.h

@@ -288,7 +288,6 @@ class IColumn;
     M(UInt64, http_response_buffer_size, 0, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \
     \
     M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0)    \
-    M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \
     \
     M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \
     \

src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp

@@ -6,7 +6,7 @@
 #include <IO/WriteHelpers.h>
 #include <IO/WriteBufferFromFileBase.h>
 #include <Common/logger_useful.h>
-
+#include <Core/ServerSettings.h>
 #include <Interpreters/Context.h>
 
 namespace DB
@@ -204,7 +204,7 @@ void DiskObjectStorageMetadata::addObject(ObjectStorageKey key, size_t size)
 bool DiskObjectStorageMetadata::getWriteFullObjectKeySetting()
 {
 #ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
-    return Context::getGlobalContextInstance()->getSettings().storage_metadata_write_full_object_key;
+    return Context::getGlobalContextInstance()->getServerSettings().storage_metadata_write_full_object_key;
 #else
     return false;
 #endif

tests/integration/test_remote_blobs_naming/configs/new_node.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<clickhouse>
+    <storage_metadata_write_full_object_key>1</storage_metadata_write_full_object_key>
+</clickhouse>

tests/integration/test_remote_blobs_naming/configs/old_node.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<clickhouse>
+    <storage_metadata_write_full_object_key>0</storage_metadata_write_full_object_key>
+</clickhouse>

tests/integration/test_remote_blobs_naming/configs/settings_new.xml

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<clickhouse>
-    <profiles>
-        <default>
-            <s3_check_objects_after_upload>1</s3_check_objects_after_upload>
-            <enable_s3_requests_logging>1</enable_s3_requests_logging>
-            <storage_metadata_write_full_object_key>1</storage_metadata_write_full_object_key>
-        </default>
-    </profiles>
-</clickhouse>

tests/integration/test_remote_blobs_naming/configs/switching_node.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<clickhouse>
+    <storage_metadata_write_full_object_key>0</storage_metadata_write_full_object_key>
+</clickhouse>

tests/integration/test_remote_blobs_naming/test_backward_compatibility.py

@@ -13,6 +13,7 @@ def cluster():
     cluster.add_instance(
         "node",
         main_configs=[
+            "configs/old_node.xml",
             "configs/storage_conf.xml",
         ],
         user_configs=[
@@ -25,10 +26,11 @@ def cluster():
     cluster.add_instance(
         "new_node",
         main_configs=[
+            "configs/new_node.xml",
             "configs/storage_conf.xml",
         ],
         user_configs=[
-            "configs/settings_new.xml",
+            "configs/settings.xml",
         ],
         with_minio=True,
         macros={"replica": "2"},
@@ -37,6 +39,7 @@ def cluster():
     cluster.add_instance(
         "switching_node",
         main_configs=[
+            "configs/switching_node.xml",
             "configs/storage_conf.xml",
         ],
         user_configs=[
@@ -238,24 +241,21 @@ def test_replicated_merge_tree(cluster, storage_policy):
 
 
 def switch_config_write_full_object_key(node, enable):
-    setting_path = "/etc/clickhouse-server/users.d/settings.xml"
+    setting_path = "/etc/clickhouse-server/config.d/switching_node.xml"
+
+    is_on = "<storage_metadata_write_full_object_key>1<"
+    is_off = "<storage_metadata_write_full_object_key>0<"
+
     data = read_file(node, setting_path)
 
     assert data != ""
+    assert is_on in data or is_off in data
 
-    is_on = "<storage_metadata_write_full_object_key>1</storage_metadata_write_full_object_key>"
-    is_off = "<storage_metadata_write_full_object_key>0</storage_metadata_write_full_object_key>"
-
-    enable_line = is_off
     if enable:
-        enable_line = is_on
-
-    if is_on in data:
-        data = data.replace(is_on, enable_line)
+        node.replace_in_config(setting_path, is_off, is_on)
     else:
-        data = data.replace(is_off, enable_line)
+        node.replace_in_config(setting_path, is_on, is_off)
 
-    write_file(node, setting_path, data)
     node.restart_clickhouse()