Merge branch 'master' of github.com:ClickHouse/ClickHouse into clickhouse-fix-msan-report-in-epoll

.github/workflows/reusable_test.yml

@@ -102,6 +102,8 @@ jobs:
             --job-name '${{inputs.test_name}}' \
             --run \
             --run-command '''${{inputs.run_command}}'''
+          # shellcheck disable=SC2319
+          echo "JOB_EXIT_CODE=$?" >> "$GITHUB_ENV"
       - name: Post run
         if: ${{ !cancelled() }}
         run: |

contrib/jemalloc-cmake/CMakeLists.txt

@@ -34,11 +34,7 @@ if (OS_LINUX)
     # avoid spurious latencies and additional work associated with
     # MADV_DONTNEED. See
     # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
-    if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
-    else()
-        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
-    endif()
+    set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
 else()
     set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
 endif()

contrib/jemalloc-cmake/README

@@ -4,3 +4,14 @@ It allows to integrate JEMalloc into CMake project.
 - Added JEMALLOC_CONFIG_MALLOC_CONF substitution
 - Add musl support (USE_MUSL)
 - Also note, that darwin build requires JEMALLOC_PREFIX, while others do not
+- JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE should be disabled
+
+  CLOCK_MONOTONIC_COARSE can go backwards after clock_adjtime(ADJ_FREQUENCY)
+  Let's disable it for now, and this menas that CLOCK_MONOTONIC will be used,
+  and this, should not be a problem, since:
+  - jemalloc do not call clock_gettime() that frequently
+  - the difference is CLOCK_MONOTONIC 20ns and CLOCK_MONOTONIC_COARSE 4ns
+
+  This can be done with the following command:
+
+      gg JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE | cut -d: -f1 | xargs sed -i 's@#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE@/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */@'

contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -96,7 +96,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ 
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -98,7 +98,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -98,7 +98,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -98,7 +98,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -96,7 +96,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */ 
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -98,7 +98,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -99,7 +99,7 @@
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
-#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
+/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
 
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

docker/test/stateful/run.sh

@@ -25,7 +25,7 @@ source /utils.lib
 azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence &
 
 ./setup_minio.sh stateful
-./mc admin trace clickminio > /test_output/rubbish.log &
+./mc admin trace clickminio > /test_output/minio.log &
 MC_ADMIN_PID=$!
 
 config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml

docker/test/stateless/run.sh

@@ -54,7 +54,7 @@ source /utils.lib
 /usr/share/clickhouse-test/config/install.sh
 
 ./setup_minio.sh stateless
-m./c admin trace clickminio > /test_output/rubbish.log &
+./mc admin trace clickminio > /test_output/minio.log &
 MC_ADMIN_PID=$!
 
 ./setup_hdfs_minicluster.sh

docker/test/stateless/setup_hdfs_minicluster.sh

@@ -10,7 +10,7 @@ cd hadoop-3.3.1
 export JAVA_HOME=/usr
 mkdir -p target/test/data
 chown clickhouse ./target/test/data
-sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/garbage.log 2>&1 &
+sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/hdfs_minicluster.log 2>&1 &
 
 while ! nc -z localhost 12222; do
   sleep 1

docs/en/sql-reference/functions/other-functions.md

@@ -2984,6 +2984,66 @@ Result:
 └─────────┘
 ```
 
+## partitionID
+
+Computes the [partition ID](../../engines/table-engines/mergetree-family/custom-partitioning-key.md).
+
+:::note
+This function is slow and should not be called for large amount of rows.
+:::
+
+**Syntax**
+
+```sql
+partitionID(x[, y, ...]);
+```
+
+**Arguments**
+
+- `x` — Column for which to return the partition ID.
+- `y, ...` — Remaining N columns for which to return the partition ID (optional).
+
+**Returned Value**
+
+- Partition ID that the row would belong to. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS tab;
+
+CREATE TABLE tab
+(
+  i int,
+  j int
+)
+ENGINE = MergeTree
+PARTITION BY i
+ORDER BY tuple();
+
+INSERT INTO tab VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6);
+
+SELECT i, j, partitionID(i), _partition_id FROM tab ORDER BY i, j;
+```
+
+Result:
+
+```response
+┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐
+│ 1 │ 1 │ 1              │ 1             │
+│ 1 │ 2 │ 1              │ 1             │
+│ 1 │ 3 │ 1              │ 1             │
+└───┴───┴────────────────┴───────────────┘
+┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐
+│ 2 │ 4 │ 2              │ 2             │
+│ 2 │ 5 │ 2              │ 2             │
+│ 2 │ 6 │ 2              │ 2             │
+└───┴───┴────────────────┴───────────────┘
+```
+
+
 ## shardNum
 
 Returns the index of a shard which processes a part of data in a distributed query. Indices are started from `1`.

programs/client/Client.cpp

@@ -24,9 +24,8 @@
 #include <Common/TerminalSize.h>
 #include <Common/config_version.h>
 #include <Common/formatReadable.h>
-
+#include <Core/Settings.h>
 #include <Columns/ColumnString.h>
-#include <Poco/Util/Application.h>
 
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
@@ -49,6 +48,8 @@
 #include <Formats/registerFormats.h>
 #include <Formats/FormatFactory.h>
 
+#include <Poco/Util/Application.h>
+
 namespace fs = std::filesystem;
 using namespace std::literals;
 

programs/library-bridge/LibraryBridge.cpp

@@ -1,5 +1,7 @@
 #include "LibraryBridge.h"
 
+#include <iostream>
+
 int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
 {
     DB::LibraryBridge app;

programs/library-bridge/LibraryBridgeHandlers.cpp

@@ -6,6 +6,7 @@
 #include "ExternalDictionaryLibraryHandlerFactory.h"
 
 #include <Formats/FormatFactory.h>
+#include <IO/Operators.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <Common/BridgeProtocolVersion.h>

programs/local/LocalServer.h

@@ -3,11 +3,12 @@
 #include <Client/ClientBase.h>
 #include <Client/LocalConnection.h>
 
-#include <Common/StatusFile.h>
-#include <Common/InterruptListener.h>
-#include <Loggers/Loggers.h>
+#include <Core/ServerSettings.h>
 #include <Core/Settings.h>
 #include <Interpreters/Context.h>
+#include <Loggers/Loggers.h>
+#include <Common/InterruptListener.h>
+#include <Common/StatusFile.h>
 
 #include <filesystem>
 #include <memory>

programs/odbc-bridge/ColumnInfoHandler.cpp

@@ -2,6 +2,7 @@
 
 #if USE_ODBC
 
+#include <Core/Settings.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>

programs/odbc-bridge/IdentifierQuoteHandler.cpp

@@ -11,6 +11,7 @@
 #include <Poco/Net/HTTPServerResponse.h>
 #include <Common/BridgeProtocolVersion.h>
 #include <Common/logger_useful.h>
+#include <Core/Settings.h>
 #include "getIdentifierQuote.h"
 #include "validateODBCConnectionString.h"
 #include "ODBCPooledConnectionFactory.h"

programs/odbc-bridge/MainHandler.cpp

@@ -9,6 +9,7 @@
 #include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
+#include <Core/Settings.h>
 #include <IO/ReadBufferFromIStream.h>
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>

programs/odbc-bridge/ODBCBridge.cpp

@@ -1,5 +1,7 @@
 #include "ODBCBridge.h"
 
+#include <iostream>
+
 int mainEntryClickHouseODBCBridge(int argc, char ** argv)
 {
     DB::ODBCBridge app;

programs/odbc-bridge/SchemaAllowedHandler.cpp

@@ -2,8 +2,10 @@
 
 #if USE_ODBC
 
+#include <Core/Settings.h>
 #include <Server/HTTP/HTMLForm.h>
 #include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
+#include <IO/Operators.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Poco/Net/HTTPServerRequest.h>

programs/server/Server.cpp

@@ -68,6 +68,7 @@
 #include <Interpreters/registerInterpreters.h>
 #include <Interpreters/JIT/CompiledExpressionCache.h>
 #include <Access/AccessControl.h>
+#include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>

src/Access/AccessControl.cpp

@@ -21,7 +21,6 @@
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/RestorerFromBackup.h>
 #include <Core/Settings.h>
-#include <Storages/MergeTree/MergeTreeSettings.h>
 #include <base/defines.h>
 #include <IO/Operators.h>
 #include <Common/re2.h>

src/AggregateFunctions/AggregateFunctionFactory.cpp

@@ -6,6 +6,7 @@
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
 #include <Common/CurrentThread.h>
+#include <Core/Settings.h>
 
 static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000;
 

src/Analyzer/InDepthQueryTreeVisitor.h

@@ -3,7 +3,6 @@
 #include <base/scope_guard.h>
 
 #include <Common/Exception.h>
-#include <Core/Settings.h>
 
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/QueryNode.h>

src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp

@@ -10,6 +10,8 @@
 #include <Analyzer/TableNode.h>
 #include <Analyzer/UnionNode.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp

@@ -11,6 +11,8 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/Utils.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/ArrayExistsToHasPass.cpp

@@ -11,6 +11,8 @@
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/LambdaNode.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/AutoFinalOnQueryPass.cpp

@@ -8,6 +8,8 @@
 #include <Analyzer/TableExpressionModifiers.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/ConvertOrLikeChainPass.cpp

@@ -22,6 +22,8 @@
 #include <Analyzer/HashUtils.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/ConvertQueryToCNFPass.cpp

@@ -10,6 +10,8 @@
 #include <Analyzer/Utils.h>
 #include <Analyzer/HashUtils.h>
 
+#include <Core/Settings.h>
+
 #include <Storages/IStorage.h>
 
 #include <Functions/FunctionFactory.h>

src/Analyzer/Passes/CountDistinctPass.cpp

@@ -11,6 +11,8 @@
 #include <Analyzer/QueryNode.h>
 #include <Analyzer/Utils.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/CrossToInnerJoinPass.cpp

@@ -9,13 +9,14 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/ConstantNode.h>
 #include <Analyzer/ColumnNode.h>
+#include <Analyzer/Utils.h>
 
 #include <Functions/FunctionFactory.h>
 #include <Functions/IFunction.h>
 #include <Functions/logical.h>
 
 #include <Common/logger_useful.h>
-#include <Analyzer/Utils.h>
+#include <Core/Settings.h>
 
 
 namespace DB

src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp

@@ -21,6 +21,8 @@
 #include <Analyzer/Utils.h>
 #include <Analyzer/JoinNode.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Passes/FuseFunctionsPass.cpp

@@ -1,6 +1,7 @@
 #include <Analyzer/Passes/FuseFunctionsPass.h>
 
 #include <Common/iota.h>
+#include <Core/Settings.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeTuple.h>

src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp

@@ -1,6 +1,7 @@
 #include <Analyzer/Passes/GroupingFunctionsResolvePass.h>
 
 #include <Core/ColumnNumbers.h>
+#include <Core/Settings.h>
 
 #include <Functions/grouping.h>
 

src/Analyzer/Passes/IfChainToMultiIfPass.cpp

@@ -4,6 +4,7 @@
 
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/FunctionNode.h>
+#include <Core/Settings.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/multiIf.h>
 

src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp

@@ -5,6 +5,7 @@
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/Utils.h>
+#include <Core/Settings.h>
 
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeEnum.h>

src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp

@@ -8,6 +8,7 @@
 #include <Analyzer/JoinNode.h>
 #include <Analyzer/HashUtils.h>
 #include <Analyzer/Utils.h>
+#include <Core/Settings.h>
 
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypesNumber.h>

src/Analyzer/Passes/MultiIfToIfPass.cpp

@@ -2,6 +2,7 @@
 
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/FunctionNode.h>
+#include <Core/Settings.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/if.h>
 

src/Analyzer/Passes/NormalizeCountVariantsPass.cpp

@@ -7,6 +7,7 @@
 #include <Analyzer/ConstantNode.h>
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/Utils.h>
+#include <Core/Settings.h>
 #include <Interpreters/Context.h>
 #include <DataTypes/DataTypesNumber.h>
 

src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp

@@ -9,6 +9,7 @@
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Common/DateLUT.h>
 #include <Common/DateLUTImpl.h>
+#include <Core/Settings.h>
 
 namespace DB
 {

src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp

@@ -8,6 +8,7 @@
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/QueryNode.h>
+#include <Core/Settings.h>
 
 namespace DB
 {

src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp

@@ -3,6 +3,7 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/IQueryTreeNode.h>
+#include <Core/Settings.h>
 #include <DataTypes/IDataType.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 

src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp

@@ -8,6 +8,7 @@
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/QueryNode.h>
 #include <Analyzer/SortNode.h>
+#include <Core/Settings.h>
 
 namespace DB
 {

src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp

@@ -6,6 +6,8 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <AggregateFunctions/IAggregateFunction.h>
 
+#include <Core/Settings.h>
+
 #include <Functions/FunctionFactory.h>
 
 #include <Interpreters/Context.h>

src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp

@@ -6,6 +6,7 @@
 #include <Analyzer/ConstantNode.h>
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/Utils.h>
+#include <Core/Settings.h>
 #include <Functions/FunctionFactory.h>
 
 namespace DB

src/Analyzer/Passes/SumIfToCountIfPass.cpp

@@ -1,19 +1,16 @@
 #include <Analyzer/Passes/SumIfToCountIfPass.h>
 
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeNullable.h>
-
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <AggregateFunctions/IAggregateFunction.h>
-#include <Analyzer/Utils.h>
-
-#include <Functions/FunctionFactory.h>
-
-#include <Interpreters/Context.h>
-
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/ConstantNode.h>
 #include <Analyzer/FunctionNode.h>
+#include <Analyzer/Utils.h>
+#include <Core/Settings.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <Functions/FunctionFactory.h>
+#include <Interpreters/Context.h>
 
 namespace DB
 {

src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp

@@ -9,6 +9,8 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/Utils.h>
 
+#include <Core/Settings.h>
+
 
 namespace DB
 {

src/Analyzer/Passes/UniqToCountPass.cpp

@@ -9,6 +9,8 @@
 #include <Analyzer/QueryNode.h>
 #include <Analyzer/Utils.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/QueryTreeBuilder.cpp

@@ -40,6 +40,8 @@
 #include <Analyzer/JoinNode.h>
 #include <Analyzer/UnionNode.h>
 
+#include <Core/Settings.h>
+
 #include <Databases/IDatabase.h>
 
 #include <Interpreters/StorageID.h>

src/Analyzer/Resolve/IdentifierResolveScope.cpp

@@ -1,8 +1,9 @@
 #include <Analyzer/Resolve/IdentifierResolveScope.h>
 
-#include <Interpreters/Context.h>
 #include <Analyzer/QueryNode.h>
 #include <Analyzer/UnionNode.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 
 namespace DB
 {

src/Analyzer/Resolve/IdentifierResolver.cpp

@@ -25,6 +25,8 @@
 #include <Analyzer/Resolve/IdentifierResolveScope.h>
 #include <Analyzer/Resolve/ReplaceColumnsVisitor.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 namespace ErrorCodes

src/Analyzer/Resolve/QueryAnalyzer.cpp

@@ -64,6 +64,8 @@
 #include <Analyzer/Resolve/TableExpressionsAliasVisitor.h>
 #include <Analyzer/Resolve/ReplaceColumnsVisitor.h>
 
+#include <Core/Settings.h>
+
 namespace ProfileEvents
 {
     extern const Event ScalarSubqueriesGlobalCacheHit;

src/Analyzer/TableNode.cpp

@@ -10,6 +10,8 @@
 
 #include <Interpreters/Context.h>
 
+#include <Core/Settings.h>
+
 namespace DB
 {
 

src/Analyzer/Utils.cpp

@@ -1,5 +1,7 @@
 #include <Analyzer/Utils.h>
 
+#include <Core/Settings.h>
+
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTSubquery.h>

src/Backups/BackupEntriesCollector.cpp

@@ -17,6 +17,7 @@
 #include <base/scope_guard.h>
 #include <base/sleep.h>
 #include <Common/escapeForFileName.h>
+#include <Core/Settings.h>
 
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>

src/Backups/BackupIO_S3.cpp

@@ -1,6 +1,7 @@
 #include <Backups/BackupIO_S3.h>
 
 #if USE_AWS_S3
+#include <Core/Settings.h>
 #include <Common/quoteString.h>
 #include <Common/threadPoolCallbackRunner.h>
 #include <Interpreters/Context.h>

src/Backups/BackupsWorker.cpp

@@ -26,6 +26,7 @@
 #include <Common/setThreadName.h>
 #include <Common/scope_guard_safe.h>
 #include <Common/ThreadPool.h>
+#include <Core/Settings.h>
 
 #include <boost/range/adaptor/map.hpp>
 

src/Backups/DDLAdjustingForBackupVisitor.cpp

@@ -1,11 +1,11 @@
 #include <Backups/DDLAdjustingForBackupVisitor.h>
+#include <Core/ServerSettings.h>
+#include <Interpreters/Context.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
-#include <Interpreters/Context.h>
-#include <Storages/StorageReplicatedMergeTree.h>
-
 #include <Parsers/formatAST.h>
+#include <Storages/StorageReplicatedMergeTree.h>
 
 
 namespace DB

src/Backups/RestorerFromBackup.cpp

@@ -23,8 +23,9 @@
 #include <Common/quoteString.h>
 #include <Common/escapeForFileName.h>
 #include <base/insertAtEnd.h>
-#include <boost/algorithm/string/join.hpp>
+#include <Core/Settings.h>
 
+#include <boost/algorithm/string/join.hpp>
 #include <boost/range/adaptor/map.hpp>
 
 #include <filesystem>

src/Backups/WithRetries.cpp

@@ -1,5 +1,7 @@
-#include <mutex>
 #include <Backups/WithRetries.h>
+#include <Core/Settings.h>
+
+#include <mutex>
 
 namespace DB
 {

src/Bridge/IBridge.cpp

@@ -8,6 +8,7 @@
 #include <Common/SensitiveDataMasker.h>
 #include <Common/StringUtils.h>
 #include <Common/logger_useful.h>
+#include <Core/Settings.h>
 #include <Formats/registerFormats.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromFile.h>

src/BridgeHelper/LibraryBridgeHelper.cpp

@@ -1,5 +1,7 @@
 #include "LibraryBridgeHelper.h"
 
+#include <Core/ServerSettings.h>
+#include <Core/Settings.h>
 #include <IO/ConnectionTimeouts.h>
 
 namespace DB

src/Client/ClientBase.h

@@ -10,6 +10,7 @@
 #include <Common/Stopwatch.h>
 #include <Common/DNSResolver.h>
 #include <Core/ExternalTable.h>
+#include <Core/Settings.h>
 #include <Poco/Util/Application.h>
 #include <Interpreters/Context.h>
 #include <Client/Suggest.h>

src/Client/ConnectionEstablisher.cpp

@@ -1,6 +1,7 @@
 #include <Client/ConnectionEstablisher.h>
 #include <Common/quoteString.h>
 #include <Common/ProfileEvents.h>
+#include <Core/Settings.h>
 
 namespace ProfileEvents
 {

src/Client/ConnectionPool.cpp

@@ -1,4 +1,5 @@
 #include <Client/ConnectionPool.h>
+#include <Core/Settings.h>
 
 #include <boost/functional/hash.hpp>
 
@@ -85,4 +86,15 @@ ConnectionPoolFactory & ConnectionPoolFactory::instance()
     return ret;
 }
 
+IConnectionPool::Entry ConnectionPool::get(const DB::ConnectionTimeouts& timeouts, const DB::Settings& settings,
+        bool force_connected)
+{
+    Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds());
+
+    if (force_connected)
+        entry->forceConnected(timeouts);
+
+    return entry;
+}
+
 }

src/Client/ConnectionPool.h

@@ -4,12 +4,13 @@
 #include <Common/Priority.h>
 #include <Client/Connection.h>
 #include <IO/ConnectionTimeouts.h>
-#include <Core/Settings.h>
 #include <base/defines.h>
 
 namespace DB
 {
 
+struct Settings;
+
 /** Interface for connection pools.
   *
   * Usage (using the usual `ConnectionPool` example)
@@ -102,15 +103,7 @@ public:
 
     Entry get(const ConnectionTimeouts & timeouts, /// NOLINT
               const Settings & settings,
-              bool force_connected = true) override
-    {
-        Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds());
-
-        if (force_connected)
-            entry->forceConnected(timeouts);
-
-        return entry;
-    }
+              bool force_connected = true) override;
 
     std::string getDescription() const
     {

src/Client/HedgedConnections.cpp

@@ -3,6 +3,7 @@
 
 #include <Client/HedgedConnections.h>
 #include <Common/ProfileEvents.h>
+#include <Core/Settings.h>
 #include <Interpreters/ClientInfo.h>
 #include <Interpreters/Context.h>
 

src/Client/HedgedConnectionsFactory.h

@@ -8,13 +8,14 @@
 #include <Common/Fiber.h>
 #include <Client/ConnectionEstablisher.h>
 #include <Client/ConnectionPoolWithFailover.h>
-#include <Core/Settings.h>
 #include <unordered_map>
 #include <memory>
 
 namespace DB
 {
 
+struct Settings;
+
 /** Class for establishing hedged connections with replicas.
   * The process of establishing connection is divided on stages, on each stage if
   * replica doesn't respond for a long time, we start establishing connection with

src/Client/MultiplexedConnections.cpp

@@ -2,6 +2,7 @@
 
 #include <Common/thread_local_rng.h>
 #include <Core/Protocol.h>
+#include <Core/Settings.h>
 #include <Interpreters/Context.h>
 #include <IO/ConnectionTimeouts.h>
 #include <IO/Operators.h>

src/Common/DateLUT.cpp

@@ -2,7 +2,9 @@
 
 #include <Interpreters/Context.h>
 #include <Common/CurrentThread.h>
+#include <Common/DateLUTImpl.h>
 #include <Common/filesystemHelpers.h>
+#include <Core/Settings.h>
 
 #include <Poco/DigestStream.h>
 #include <Poco/Exception.h>

src/Common/GetPriorityForLoadBalancing.cpp

@@ -1,3 +1,4 @@
+#include <Common/Exception.h>
 #include <Common/GetPriorityForLoadBalancing.h>
 #include <Common/Priority.h>
 

src/Common/GetPriorityForLoadBalancing.h

@@ -1,6 +1,10 @@
 #pragma once
 
-#include <Core/SettingsEnums.h>
+#include <Common/Priority.h>
+#include <Core/LoadBalancing.h>
+
+#include <functional>
+#include <vector>
 
 namespace DB
 {

src/Common/NamedCollections/NamedCollectionsFactory.cpp

@@ -1,6 +1,7 @@
 #include <Common/NamedCollections/NamedCollectionsFactory.h>
 #include <Common/NamedCollections/NamedCollectionConfiguration.h>
 #include <Common/NamedCollections/NamedCollectionsMetadataStorage.h>
+#include <Core/Settings.h>
 #include <base/sleep.h>
 
 namespace DB

src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp

@@ -5,6 +5,7 @@
 #include <Common/ZooKeeper/IKeeper.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
+#include <Core/Settings.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/WriteHelpers.h>

src/Common/ProgressIndication.cpp

@@ -1,6 +1,7 @@
 #include "ProgressIndication.h"
 #include <algorithm>
 #include <cstddef>
+#include <iostream>
 #include <numeric>
 #include <filesystem>
 #include <cmath>

src/Common/ProgressIndication.h

@@ -1,14 +1,15 @@
 #pragma once
 
-#include <unordered_map>
-#include <unordered_set>
-#include <mutex>
 #include <IO/Progress.h>
 #include <Interpreters/Context.h>
 #include <base/types.h>
 #include <Common/Stopwatch.h>
 #include <Common/EventRateMeter.h>
 
+#include <iostream>
+#include <mutex>
+#include <unordered_map>
+#include <unordered_set>
 
 namespace DB
 {

src/Common/ThreadFuzzer.cpp

@@ -12,13 +12,14 @@
 #include <base/sleep.h>
 
 #include <IO/ReadHelpers.h>
-#include <Common/logger_useful.h>
 
+#include <Common/CurrentMemoryTracker.h>
 #include <Common/Exception.h>
 #include <Common/MemoryTracker.h>
+#include <Common/ThreadFuzzer.h>
+#include <Common/logger_useful.h>
 #include <Common/thread_local_rng.h>
 
-#include <Common/ThreadFuzzer.h>
 #include "config.h" // USE_JEMALLOC
 
 

src/Coordination/FourLetterCommand.cpp

@@ -1,5 +1,6 @@
 #include <Coordination/FourLetterCommand.h>
 
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperDispatcher.h>
 #include <Server/KeeperTCPHandler.h>
 #include <Common/ZooKeeper/IKeeper.h>

src/Coordination/KeeperContext.cpp

@@ -3,6 +3,7 @@
 
 #include <Coordination/KeeperContext.h>
 
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/Defines.h>
 #include <Coordination/KeeperConstants.h>
 #include <Server/CloudPlacementInfo.h>

src/Coordination/KeeperDispatcher.cpp

@@ -5,8 +5,8 @@
 #include <Poco/Util/AbstractConfiguration.h>
 
 #include <base/hex.h>
-#include "Common/ZooKeeper/IKeeper.h"
-#include "Common/ZooKeeper/ZooKeeperCommon.h"
+#include <Common/ZooKeeper/IKeeper.h>
+#include <Common/ZooKeeper/ZooKeeperCommon.h>
 #include <Common/setThreadName.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/checkStackSize.h>
@@ -14,6 +14,7 @@
 #include <Common/ProfileEvents.h>
 #include <Common/logger_useful.h>
 #include <Common/formatReadable.h>
+#include <Coordination/CoordinationSettings.h>
 
 #include <Disks/IDisk.h>
 

src/Coordination/KeeperDispatcher.h

@@ -11,7 +11,6 @@
 #include <Common/Exception.h>
 #include <functional>
 #include <Coordination/KeeperServer.h>
-#include <Coordination/CoordinationSettings.h>
 #include <Coordination/Keeper4LWInfo.h>
 #include <Coordination/KeeperConnectionStats.h>
 #include <Coordination/KeeperSnapshotManagerS3.h>

src/Coordination/KeeperServer.cpp

@@ -3,9 +3,7 @@
 
 #include "config.h"
 
-#include <chrono>
-#include <mutex>
-#include <string>
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperLogStore.h>
 #include <Coordination/KeeperSnapshotManagerS3.h>
 #include <Coordination/KeeperStateMachine.h>
@@ -30,6 +28,10 @@
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 
+#include <chrono>
+#include <mutex>
+#include <string>
+
 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
 #include <fmt/chrono.h>
 

src/Coordination/KeeperServer.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <Coordination/CoordinationSettings.h>
 #include <Coordination/InMemoryLogStore.h>
 #include <Coordination/KeeperStateMachine.h>
 #include <Coordination/KeeperStateManager.h>
@@ -17,6 +16,9 @@ namespace DB
 
 using RaftAppendResult = nuraft::ptr<nuraft::cmd_result<nuraft::ptr<nuraft::buffer>>>;
 
+struct KeeperConfigurationAndSettings;
+using KeeperConfigurationAndSettingsPtr = std::shared_ptr<KeeperConfigurationAndSettings>;
+
 class KeeperServer
 {
 private:

src/Coordination/KeeperStateMachine.cpp

@@ -1,6 +1,7 @@
 #include <atomic>
 #include <cerrno>
 #include <chrono>
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperDispatcher.h>
 #include <Coordination/KeeperReconfiguration.h>
 #include <Coordination/KeeperSnapshotManager.h>

src/Coordination/KeeperStateMachine.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperSnapshotManager.h>
 #include <Coordination/KeeperSnapshotManagerS3.h>
 #include <Coordination/KeeperContext.h>

src/Coordination/KeeperStateManager.h

@@ -1,14 +1,13 @@
 #pragma once
 
-#include <Core/Types.h>
 #include <string>
 #include <Coordination/KeeperLogStore.h>
-#include <Coordination/CoordinationSettings.h>
+#include <Coordination/KeeperSnapshotManager.h>
+#include <Core/Types.h>
 #include <libnuraft/nuraft.hxx>
 #include <Poco/Util/AbstractConfiguration.h>
 #include "Coordination/KeeperStateMachine.h"
 #include "Coordination/RaftServerConfig.h"
-#include <Coordination/KeeperSnapshotManager.h>
 
 namespace DB
 {

src/Coordination/KeeperStorage.cpp

@@ -20,11 +20,12 @@
 #include <Common/LockMemoryExceptionInThread.h>
 #include <Common/ProfileEvents.h>
 
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperCommon.h>
 #include <Coordination/KeeperConstants.h>
+#include <Coordination/KeeperDispatcher.h>
 #include <Coordination/KeeperReconfiguration.h>
 #include <Coordination/KeeperStorage.h>
-#include <Coordination/KeeperDispatcher.h>
 
 #include <functional>
 #include <base/defines.h>

src/Coordination/tests/gtest_coordination.cpp

@@ -7,6 +7,7 @@
 #include <filesystem>
 #include <thread>
 #include <Coordination/Changelog.h>
+#include <Coordination/CoordinationSettings.h>
 #include <Coordination/InMemoryLogStore.h>
 #include <Coordination/KeeperContext.h>
 #include <Coordination/KeeperConstants.h>

src/Core/ExternalTable.cpp

@@ -16,6 +16,7 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 
 #include <Core/ExternalTable.h>
+#include <Core/Settings.h>
 #include <Poco/Net/MessageHeader.h>
 #include <Parsers/ASTNameTypePair.h>
 #include <Parsers/ParserCreateQuery.h>

src/Core/LoadBalancing.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <cstdint>
+
+namespace DB
+{
+
+enum class LoadBalancing : uint8_t
+{
+    /// among replicas with a minimum number of errors selected randomly
+    RANDOM = 0,
+    /// a replica is selected among the replicas with the minimum number of errors
+    /// with the minimum number of distinguished characters in the replica name prefix and local hostname prefix
+    NEAREST_HOSTNAME,
+    /// just like NEAREST_HOSTNAME, but it count distinguished characters in a levenshtein distance manner
+    HOSTNAME_LEVENSHTEIN_DISTANCE,
+    // replicas with the same number of errors are accessed in the same order
+    // as they are specified in the configuration.
+    IN_ORDER,
+    /// if first replica one has higher number of errors,
+    ///   pick a random one from replicas with minimum number of errors
+    FIRST_OR_RANDOM,
+    // round robin across replicas with the same number of errors.
+    ROUND_ROBIN,
+};
+
+}

src/Core/QueryLogElementType.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <cstdint>
+
+namespace DB
+{
+
+// Make it signed for compatibility with DataTypeEnum8
+enum QueryLogElementType : int8_t
+{
+    QUERY_START = 1,
+    QUERY_FINISH = 2,
+    EXCEPTION_BEFORE_START = 3,
+    EXCEPTION_WHILE_PROCESSING = 4,
+};
+
+}

src/Core/SchemaInferenceMode.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <cstdint>
+
+namespace DB
+{
+
+enum class SchemaInferenceMode : uint8_t
+{
+    DEFAULT,
+    UNION,
+};
+
+}

src/Core/SettingsEnums.h

@@ -1,8 +1,12 @@
 #pragma once
 
 #include <Core/Joins.h>
+#include <Core/LoadBalancing.h>
 #include <Core/LogsLevel.h>
+#include <Core/QueryLogElementType.h>
+#include <Core/SchemaInferenceMode.h>
 #include <Core/SettingsFields.h>
+#include <Core/ShortCircuitFunctionEvaluation.h>
 #include <Formats/FormatSettings.h>
 #include <IO/ReadSettings.h>
 #include <Parsers/ASTSQLSecurity.h>
@@ -115,25 +119,6 @@ constexpr auto getEnumValues();
         return getEnumValues<EnumType>().size();\
     }
 
-enum class LoadBalancing : uint8_t
-{
-    /// among replicas with a minimum number of errors selected randomly
-    RANDOM = 0,
-    /// a replica is selected among the replicas with the minimum number of errors
-    /// with the minimum number of distinguished characters in the replica name prefix and local hostname prefix
-    NEAREST_HOSTNAME,
-    /// just like NEAREST_HOSTNAME, but it count distinguished characters in a levenshtein distance manner
-    HOSTNAME_LEVENSHTEIN_DISTANCE,
-    // replicas with the same number of errors are accessed in the same order
-    // as they are specified in the configuration.
-    IN_ORDER,
-    /// if first replica one has higher number of errors,
-    ///   pick a random one from replicas with minimum number of errors
-    FIRST_OR_RANDOM,
-    // round robin across replicas with the same number of errors.
-    ROUND_ROBIN,
-};
-
 DECLARE_SETTING_ENUM(LoadBalancing)
 
 DECLARE_SETTING_ENUM(JoinStrictness)
@@ -204,16 +189,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion)
 
 DECLARE_SETTING_ENUM(LogsLevel)
 
-
-// Make it signed for compatibility with DataTypeEnum8
-enum QueryLogElementType : int8_t
-{
-    QUERY_START = 1,
-    QUERY_FINISH = 2,
-    EXCEPTION_BEFORE_START = 3,
-    EXCEPTION_WHILE_PROCESSING = 4,
-};
-
 DECLARE_SETTING_ENUM_WITH_RENAME(LogQueriesType, QueryLogElementType)
 
 
@@ -292,13 +267,6 @@ enum class StreamingHandleErrorMode : uint8_t
 
 DECLARE_SETTING_ENUM(StreamingHandleErrorMode)
 
-enum class ShortCircuitFunctionEvaluation : uint8_t
-{
-    ENABLE, // Use short-circuit function evaluation for functions that are suitable for it.
-    FORCE_ENABLE, // Use short-circuit function evaluation for all functions.
-    DISABLE, // Disable short-circuit function evaluation.
-};
-
 DECLARE_SETTING_ENUM(ShortCircuitFunctionEvaluation)
 
 enum class TransactionsWaitCSNMode : uint8_t
@@ -367,12 +335,6 @@ DECLARE_SETTING_ENUM(ObjectStorageQueueAction)
 
 DECLARE_SETTING_ENUM(ExternalCommandStderrReaction)
 
-enum class SchemaInferenceMode : uint8_t
-{
-    DEFAULT,
-    UNION,
-};
-
 DECLARE_SETTING_ENUM(SchemaInferenceMode)
 
 DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeOverflowBehavior, FormatSettings::DateTimeOverflowBehavior)

src/Core/ShortCircuitFunctionEvaluation.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <cstdint>
+
+namespace DB
+{
+
+enum class ShortCircuitFunctionEvaluation : uint8_t
+{
+    ENABLE, // Use short-circuit function evaluation for functions that are suitable for it.
+    FORCE_ENABLE, // Use short-circuit function evaluation for all functions.
+    DISABLE, // Disable short-circuit function evaluation.
+};
+
+}

src/Core/SortDescription.h

@@ -1,15 +1,16 @@
 #pragma once
 
-#include <vector>
-#include <memory>
-#include <cstddef>
-#include <string>
+
 #include <Core/Field.h>
-#include <Core/SettingsEnums.h>
 #include <Common/IntervalKind.h>
 #include <DataTypes/IDataType.h>
 #include <Columns/Collator.h>
 
+#include <cstddef>
+#include <string>
+#include <memory>
+#include <vector>
+
 namespace DB
 {
 

src/Daemon/BaseDaemon.cpp

@@ -4,6 +4,7 @@
 #include <base/errnoToString.h>
 #include <Common/CurrentThread.h>
 #include <Common/MemoryTracker.h>
+#include <Core/Settings.h>
 #include <Daemon/BaseDaemon.h>
 #include <Daemon/SentryWriter.h>
 #include <Common/GWPAsan.h>