Merge pull request #37326 from ClickHouse/fix-marks-cache-size-cfg

Provide default value for mark_cache_size

programs/local/LocalServer.cpp

@@ -544,8 +544,7 @@ void LocalServer::processConfig()
     if (uncompressed_cache_size)
         global_context->setUncompressedCache(uncompressed_cache_size);
 
-    /// Size of cache for marks (index of MergeTree family of tables). It is necessary.
+    /// Size of cache for marks (index of MergeTree family of tables).
-    /// Specify default value for mark_cache_size explicitly!
     size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
     if (mark_cache_size)
         global_context->setMarkCache(mark_cache_size);
@@ -555,8 +554,7 @@ void LocalServer::processConfig()
     if (index_uncompressed_cache_size)
         global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
 
-    /// Size of cache for index marks (index of MergeTree skip indices). It is necessary.
+    /// Size of cache for index marks (index of MergeTree skip indices).
-    /// Specify default value for index_mark_cache_size explicitly!
     size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", 0);
     if (index_mark_cache_size)
         global_context->setIndexMarkCache(index_mark_cache_size);

programs/server/Server.cpp

@@ -1351,8 +1351,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
             settings.async_insert_max_data_size,
             AsynchronousInsertQueue::Timeout{.busy = settings.async_insert_busy_timeout_ms, .stale = settings.async_insert_stale_timeout_ms}));
 
-    /// Size of cache for marks (index of MergeTree family of tables). It is mandatory.
+    /// Size of cache for marks (index of MergeTree family of tables).
-    size_t mark_cache_size = config().getUInt64("mark_cache_size");
+    size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
     if (!mark_cache_size)
         LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
     if (mark_cache_size > max_cache_size)
@@ -1368,8 +1368,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     if (index_uncompressed_cache_size)
         global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
 
-    /// Size of cache for index marks (index of MergeTree skip indices). It is necessary.
+    /// Size of cache for index marks (index of MergeTree skip indices).
-    /// Specify default value for index_mark_cache_size explicitly!
     size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", 0);
     if (index_mark_cache_size)
         global_context->setIndexMarkCache(index_mark_cache_size);

programs/server/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_config_corresponding_root/configs/config.xml

@@ -101,13 +101,6 @@
       -->
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
 
-    <!-- Approximate size of mark cache, used in tables of MergeTree family.
-         In bytes. Cache is single for server. Memory is allocated only on demand.
-         You should not lower this value.
-      -->
-    <mark_cache_size>5368709120</mark_cache_size>
-
-
     <!-- Path to data directory, with trailing slash. -->
     <path>/var/lib/clickhouse/</path>
 

tests/integration/test_config_xml_full/configs/config.xml

@@ -304,12 +304,6 @@
       -->
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
 
-    <!-- Approximate size of mark cache, used in tables of MergeTree family.
-         In bytes. Cache is single for server. Memory is allocated only on demand.
-         You should not lower this value.
-      -->
-    <mark_cache_size>5368709120</mark_cache_size>
-
 
     <!-- If you enable the `min_bytes_to_use_mmap_io` setting,
          the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace.

tests/integration/test_config_xml_full/configs/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_config_xml_main/configs/config.xml

@@ -60,8 +60,6 @@
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
 
-    <mark_cache_size>5368709120</mark_cache_size>
-
     <mmap_cache_size>1000</mmap_cache_size>
 
     <path>./</path>

tests/integration/test_config_xml_main/configs/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_config_xml_yaml_mix/configs/config.xml

@@ -60,8 +60,6 @@
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
 
-    <mark_cache_size>5368709120</mark_cache_size>
-
     <mmap_cache_size>1000</mmap_cache_size>
 
     <path>./</path>

tests/integration/test_config_xml_yaml_mix/configs/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_config_yaml_full/configs/config.yaml

@@ -46,7 +46,6 @@ max_server_memory_usage_to_ram_ratio: 0.9
 total_memory_profiler_step: 4194304
 total_memory_tracker_sample_probability: 0
 uncompressed_cache_size: 8589934592
-mark_cache_size: 5368709120
 mmap_cache_size: 1000
 path: ./
 tmp_path: ./tmp

tests/integration/test_config_yaml_full/configs/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_config_yaml_main/configs/config.yaml

@@ -46,7 +46,6 @@ max_server_memory_usage_to_ram_ratio: 0.9
 total_memory_profiler_step: 4194304
 total_memory_tracker_sample_probability: 0
 uncompressed_cache_size: 8589934592
-mark_cache_size: 5368709120
 mmap_cache_size: 1000
 path: ./
 tmp_path: ./tmp/

tests/integration/test_config_yaml_main/configs/embedded.xml

@@ -13,7 +13,6 @@
     <path>./</path>
 
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
-    <mark_cache_size>5368709120</mark_cache_size>
     <mlock_executable>true</mlock_executable>
 
     <users>

tests/integration/test_dictionaries_all_layouts_separate_sources/configs/config.xml

@@ -24,7 +24,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_complex_key_cache_string/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_dependency_xml/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_mysql/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_null_value/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_select_all/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_update_and_reload/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionaries_update_field/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionary_allow_read_expired_keys/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_dictionary_custom_settings/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_disk_types/configs/config.xml

@@ -41,7 +41,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_https_replication/configs/config.xml

@@ -94,13 +94,6 @@
       -->
     <uncompressed_cache_size>8589934592</uncompressed_cache_size>
 
-    <!-- Approximate size of mark cache, used in tables of MergeTree family.
-         In bytes. Cache is single for server. Memory is allocated only on demand.
-         You should not lower this value.
-      -->
-    <mark_cache_size>5368709120</mark_cache_size>
-
-
     <!-- Path to data directory, with trailing slash. -->
     <path>/var/lib/clickhouse/</path>
 

tests/integration/test_join_set_family_s3/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_log_family_s3/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_logs_level/configs/config_information.xml

@@ -19,7 +19,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <users_config>users.xml</users_config>
 
     <dictionaries_config>/etc/clickhouse-server/config.d/*.xml</dictionaries_config>

tests/integration/test_match_process_uid_against_data_owner/configs/config.xml

@@ -12,6 +12,5 @@
 
     <path>/var/lib/clickhouse/</path>
 
-    <mark_cache_size>5368709120</mark_cache_size>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_merge_tree_azure_blob_storage/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_merge_tree_s3/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_merge_tree_s3_failover/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_merge_tree_s3_restore/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_merge_tree_s3_with_cache/configs/config.xml

@@ -21,7 +21,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>0</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_mysql_protocol/configs/config.xml

@@ -29,7 +29,6 @@
     <listen_host>127.0.0.1</listen_host>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_odbc_interaction/configs/config.xml

@@ -28,7 +28,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_postgresql_protocol/configs/config.xml

@@ -29,7 +29,6 @@
      <listen_host>127.0.0.1</listen_host>
 
      <max_concurrent_queries>500</max_concurrent_queries>
-     <mark_cache_size>5368709120</mark_cache_size>
      <path>./clickhouse/</path>
      <users_config>users.xml</users_config>
  </clickhouse>

tests/integration/test_profile_events_s3/configs/config.xml

@@ -29,7 +29,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_reload_auxiliary_zookeepers/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_reload_max_table_size_to_drop/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_reload_zookeeper/configs/config.xml

@@ -22,7 +22,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 

tests/integration/test_s3_with_https/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>

tests/integration/test_s3_with_proxy/configs/config.xml

@@ -14,7 +14,6 @@
     </openSSL>
 
     <max_concurrent_queries>500</max_concurrent_queries>
-    <mark_cache_size>5368709120</mark_cache_size>
     <path>./clickhouse/</path>
     <users_config>users.xml</users_config>
 </clickhouse>