Merge remote-tracking branch 'origin/master' into pr-local-plan

This commit is contained in:
Igor Nikonov 2024-07-15 12:04:04 +00:00
commit fdbafdb8db
477 changed files with 1400 additions and 607 deletions

View File

@ -13,3 +13,6 @@
# dbms/ → src/
# (though it is unlikely that you will see it in blame)
06446b4f08a142d6f1bc30664c47ded88ab51782
# Applied Black formatter for Python code
e6f5a3f98b21ba99cf274a9833797889e020a2b3

View File

@ -34,11 +34,7 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
endif()
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
endif()

View File

@ -4,3 +4,14 @@ It allows to integrate JEMalloc into CMake project.
- Added JEMALLOC_CONFIG_MALLOC_CONF substitution
- Add musl support (USE_MUSL)
- Also note, that darwin build requires JEMALLOC_PREFIX, while others do not
- JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE should be disabled
CLOCK_MONOTONIC_COARSE can go backwards after clock_adjtime(ADJ_FREQUENCY)
Let's disable it for now, and this menas that CLOCK_MONOTONIC will be used,
and this, should not be a problem, since:
- jemalloc do not call clock_gettime() that frequently
- the difference is CLOCK_MONOTONIC 20ns and CLOCK_MONOTONIC_COARSE 4ns
This can be done with the following command:
gg JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE | cut -d: -f1 | xargs sed -i 's@#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE@/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */@'

View File

@ -96,7 +96,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -98,7 +98,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -98,7 +98,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -98,7 +98,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -96,7 +96,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -98,7 +98,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

View File

@ -99,7 +99,7 @@
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.

2
contrib/openssl vendored

@ -1 +1 @@
Subproject commit ee2bb8513b28bf86b35404dd17a0e29305ca9e08
Subproject commit 66deddc1e53cda8706604a019777259372d1bd62

View File

@ -226,15 +226,59 @@ Other IDEs you can use are [Sublime Text](https://www.sublimetext.com/), [Visual
## Writing Code {#writing-code}
The description of ClickHouse architecture can be found here: https://clickhouse.com/docs/en/development/architecture/
Below you can find some quick links which may be useful when writing code for ClickHouse:
The Code Style Guide: https://clickhouse.com/docs/en/development/style/
- [ClickHouse architecture description](https://clickhouse.com/docs/en/development/architecture/).
- [The code style guide](https://clickhouse.com/docs/en/development/style/).
- [Adding third-party libraries](https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries)
- [Writing tests](https://clickhouse.com/docs/en/development/tests/)
- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest)
Adding third-party libraries: https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries
## Writing Documentation {#writing-documentation}
Writing tests: https://clickhouse.com/docs/en/development/tests/
As part of every pull request which adds a new feature, it is necessary to write documentation for it. If you'd like to preview your documentation changes the instructions for how to build the documentation page locally are available in the README.md file [here](https://github.com/ClickHouse/clickhouse-docs). When adding a new function to ClickHouse you can use the template below as a guide:
List of tasks: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest
```markdown
# newFunctionName
A short description of the function goes here. It should describe briefly what it does and a typical usage case.
**Syntax**
\```sql
newFunctionName(arg1, arg2[, arg3])
\```
**Arguments**
- `arg1` — Description of the argument. [DataType](../data-types/float.md)
- `arg2` — Description of the argument. [DataType](../data-types/float.md)
- `arg3` — Description of optional argument (optional). [DataType](../data-types/float.md)
**Implementation Details**
A description of implementation details if relevant.
**Returned value**
- Returns {insert what the function returns here}. [DataType](../data-types/float.md)
**Example**
Query:
\```sql
SELECT 'write your example query here';
\```
Response:
\```response
┌───────────────────────────────────┐
│ the result of the query │
└───────────────────────────────────┘
\```
```
## Test Data {#test-data}

View File

@ -15,7 +15,7 @@ You have four options for getting up and running with ClickHouse:
- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse
- **[Quick Install](#quick-install):** an easy-to-download binary for testing and developing with ClickHouse
- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, ARM, or PowerPC64LE CPU architecture
- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, modern ARM (ARMv8.2-A up), or PowerPC64LE CPU architecture
- **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** use the official Docker image in Docker Hub
## ClickHouse Cloud

View File

@ -2984,6 +2984,66 @@ Result:
└─────────┘
```
## partitionID
Computes the [partition ID](../../engines/table-engines/mergetree-family/custom-partitioning-key.md).
:::note
This function is slow and should not be called for large amount of rows.
:::
**Syntax**
```sql
partitionID(x[, y, ...]);
```
**Arguments**
- `x` — Column for which to return the partition ID.
- `y, ...` — Remaining N columns for which to return the partition ID (optional).
**Returned Value**
- Partition ID that the row would belong to. [String](../data-types/string.md).
**Example**
Query:
```sql
DROP TABLE IF EXISTS tab;
CREATE TABLE tab
(
i int,
j int
)
ENGINE = MergeTree
PARTITION BY i
ORDER BY tuple();
INSERT INTO tab VALUES (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6);
SELECT i, j, partitionID(i), _partition_id FROM tab ORDER BY i, j;
```
Result:
```response
┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐
│ 1 │ 1 │ 1 │ 1 │
│ 1 │ 2 │ 1 │ 1 │
│ 1 │ 3 │ 1 │ 1 │
└───┴───┴────────────────┴───────────────┘
┌─i─┬─j─┬─partitionID(i)─┬─_partition_id─┐
│ 2 │ 4 │ 2 │ 2 │
│ 2 │ 5 │ 2 │ 2 │
│ 2 │ 6 │ 2 │ 2 │
└───┴───┴────────────────┴───────────────┘
```
## shardNum
Returns the index of a shard which processes a part of data in a distributed query. Indices are started from `1`.

View File

@ -24,9 +24,8 @@
#include <Common/TerminalSize.h>
#include <Common/config_version.h>
#include <Common/formatReadable.h>
#include <Core/Settings.h>
#include <Columns/ColumnString.h>
#include <Poco/Util/Application.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
@ -49,6 +48,8 @@
#include <Formats/registerFormats.h>
#include <Formats/FormatFactory.h>
#include <Poco/Util/Application.h>
namespace fs = std::filesystem;
using namespace std::literals;

View File

@ -1,5 +1,7 @@
#include "LibraryBridge.h"
#include <iostream>
int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
{
DB::LibraryBridge app;

View File

@ -6,6 +6,7 @@
#include "ExternalDictionaryLibraryHandlerFactory.h"
#include <Formats/FormatFactory.h>
#include <IO/Operators.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
#include <Common/BridgeProtocolVersion.h>

View File

@ -3,11 +3,12 @@
#include <Client/ClientBase.h>
#include <Client/LocalConnection.h>
#include <Common/StatusFile.h>
#include <Common/InterruptListener.h>
#include <Loggers/Loggers.h>
#include <Core/ServerSettings.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <Loggers/Loggers.h>
#include <Common/InterruptListener.h>
#include <Common/StatusFile.h>
#include <filesystem>
#include <memory>

View File

@ -2,6 +2,7 @@
#if USE_ODBC
#include <Core/Settings.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeNullable.h>
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>

View File

@ -11,6 +11,7 @@
#include <Poco/Net/HTTPServerResponse.h>
#include <Common/BridgeProtocolVersion.h>
#include <Common/logger_useful.h>
#include <Core/Settings.h>
#include "getIdentifierQuote.h"
#include "validateODBCConnectionString.h"
#include "ODBCPooledConnectionFactory.h"

View File

@ -9,6 +9,7 @@
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <Core/Settings.h>
#include <IO/ReadBufferFromIStream.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>

View File

@ -1,5 +1,7 @@
#include "ODBCBridge.h"
#include <iostream>
int mainEntryClickHouseODBCBridge(int argc, char ** argv)
{
DB::ODBCBridge app;

View File

@ -2,8 +2,10 @@
#if USE_ODBC
#include <Core/Settings.h>
#include <Server/HTTP/HTMLForm.h>
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
#include <IO/Operators.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Poco/Net/HTTPServerRequest.h>

View File

@ -68,6 +68,7 @@
#include <Interpreters/registerInterpreters.h>
#include <Interpreters/JIT/CompiledExpressionCache.h>
#include <Access/AccessControl.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/System/attachSystemTables.h>
#include <Storages/System/attachInformationSchemaTables.h>

View File

@ -21,7 +21,6 @@
#include <Backups/BackupEntriesCollector.h>
#include <Backups/RestorerFromBackup.h>
#include <Core/Settings.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <base/defines.h>
#include <IO/Operators.h>
#include <Common/re2.h>

View File

@ -6,6 +6,7 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Common/CurrentThread.h>
#include <Core/Settings.h>
static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000;

View File

@ -3,7 +3,6 @@
#include <base/scope_guard.h>
#include <Common/Exception.h>
#include <Core/Settings.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/QueryNode.h>

View File

@ -10,6 +10,8 @@
#include <Analyzer/TableNode.h>
#include <Analyzer/UnionNode.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -11,6 +11,8 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -11,6 +11,8 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/LambdaNode.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -8,6 +8,8 @@
#include <Analyzer/TableExpressionModifiers.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -22,6 +22,8 @@
#include <Analyzer/HashUtils.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -10,6 +10,8 @@
#include <Analyzer/Utils.h>
#include <Analyzer/HashUtils.h>
#include <Core/Settings.h>
#include <Storages/IStorage.h>
#include <Functions/FunctionFactory.h>

View File

@ -11,6 +11,8 @@
#include <Analyzer/QueryNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -9,13 +9,14 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/Utils.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Functions/logical.h>
#include <Common/logger_useful.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
namespace DB

View File

@ -21,6 +21,8 @@
#include <Analyzer/Utils.h>
#include <Analyzer/JoinNode.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -1,6 +1,7 @@
#include <Analyzer/Passes/FuseFunctionsPass.h>
#include <Common/iota.h>
#include <Core/Settings.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>

View File

@ -1,6 +1,7 @@
#include <Analyzer/Passes/GroupingFunctionsResolvePass.h>
#include <Core/ColumnNumbers.h>
#include <Core/Settings.h>
#include <Functions/grouping.h>

View File

@ -4,6 +4,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/FunctionNode.h>
#include <Core/Settings.h>
#include <Functions/FunctionFactory.h>
#include <Functions/multiIf.h>

View File

@ -5,6 +5,7 @@
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeEnum.h>

View File

@ -8,6 +8,7 @@
#include <Analyzer/JoinNode.h>
#include <Analyzer/HashUtils.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypesNumber.h>

View File

@ -2,6 +2,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/FunctionNode.h>
#include <Core/Settings.h>
#include <Functions/FunctionFactory.h>
#include <Functions/if.h>

View File

@ -7,6 +7,7 @@
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <DataTypes/DataTypesNumber.h>

View File

@ -9,6 +9,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Common/DateLUT.h>
#include <Common/DateLUTImpl.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -8,6 +8,7 @@
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/QueryNode.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -3,6 +3,7 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Core/Settings.h>
#include <DataTypes/IDataType.h>
#include <Interpreters/ExternalDictionariesLoader.h>

View File

@ -8,6 +8,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/SortNode.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -6,6 +6,8 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <Core/Settings.h>
#include <Functions/FunctionFactory.h>
#include <Interpreters/Context.h>

View File

@ -6,6 +6,7 @@
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <Functions/FunctionFactory.h>
namespace DB

View File

@ -1,19 +1,16 @@
#include <Analyzer/Passes/SumIfToCountIfPass.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeNullable.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <Analyzer/Utils.h>
#include <Functions/FunctionFactory.h>
#include <Interpreters/Context.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeNullable.h>
#include <Functions/FunctionFactory.h>
#include <Interpreters/Context.h>
namespace DB
{

View File

@ -9,6 +9,8 @@
#include <Analyzer/FunctionNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -9,6 +9,8 @@
#include <Analyzer/QueryNode.h>
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -40,6 +40,8 @@
#include <Analyzer/JoinNode.h>
#include <Analyzer/UnionNode.h>
#include <Core/Settings.h>
#include <Databases/IDatabase.h>
#include <Interpreters/StorageID.h>

View File

@ -1,8 +1,9 @@
#include <Analyzer/Resolve/IdentifierResolveScope.h>
#include <Interpreters/Context.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/UnionNode.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
namespace DB
{

View File

@ -25,6 +25,8 @@
#include <Analyzer/Resolve/IdentifierResolveScope.h>
#include <Analyzer/Resolve/ReplaceColumnsVisitor.h>
#include <Core/Settings.h>
namespace DB
{
namespace ErrorCodes

View File

@ -64,6 +64,8 @@
#include <Analyzer/Resolve/TableExpressionsAliasVisitor.h>
#include <Analyzer/Resolve/ReplaceColumnsVisitor.h>
#include <Core/Settings.h>
namespace ProfileEvents
{
extern const Event ScalarSubqueriesGlobalCacheHit;
@ -1490,6 +1492,10 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes &
additional_column_qualification_parts = {table_expression_node->getAlias()};
else if (auto * table_node = table_expression_node->as<TableNode>())
additional_column_qualification_parts = {table_node->getStorageID().getDatabaseName(), table_node->getStorageID().getTableName()};
else if (auto * query_node = table_expression_node->as<QueryNode>(); query_node && query_node->isCTE())
additional_column_qualification_parts = {query_node->getCTEName()};
else if (auto * union_node = table_expression_node->as<UnionNode>(); union_node && union_node->isCTE())
additional_column_qualification_parts = {union_node->getCTEName()};
size_t additional_column_qualification_parts_size = additional_column_qualification_parts.size();
const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node);
@ -4455,9 +4461,8 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
{
auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree());
if (table_expression_node.get() == left_table_expression.get() &&
scope.joins_count == 1 &&
scope.context->getSettingsRef().single_join_prefer_left_table)
table_expression_data.should_qualify_columns = false;
scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table)
table_expression_data.should_qualify_columns = false;
}
scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data));

View File

@ -10,6 +10,8 @@
#include <Interpreters/Context.h>
#include <Core/Settings.h>
namespace DB
{

View File

@ -1,5 +1,7 @@
#include <Analyzer/Utils.h>
#include <Core/Settings.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTSubquery.h>

View File

@ -17,6 +17,7 @@
#include <base/scope_guard.h>
#include <base/sleep.h>
#include <Common/escapeForFileName.h>
#include <Core/Settings.h>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/algorithm/copy.hpp>

View File

@ -1,6 +1,7 @@
#include <Backups/BackupIO_S3.h>
#if USE_AWS_S3
#include <Core/Settings.h>
#include <Common/quoteString.h>
#include <Common/threadPoolCallbackRunner.h>
#include <Interpreters/Context.h>

View File

@ -26,6 +26,7 @@
#include <Common/setThreadName.h>
#include <Common/scope_guard_safe.h>
#include <Common/ThreadPool.h>
#include <Core/Settings.h>
#include <boost/range/adaptor/map.hpp>

View File

@ -1,11 +1,11 @@
#include <Backups/DDLAdjustingForBackupVisitor.h>
#include <Core/ServerSettings.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Interpreters/Context.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Parsers/formatAST.h>
#include <Storages/StorageReplicatedMergeTree.h>
namespace DB

View File

@ -23,8 +23,9 @@
#include <Common/quoteString.h>
#include <Common/escapeForFileName.h>
#include <base/insertAtEnd.h>
#include <boost/algorithm/string/join.hpp>
#include <Core/Settings.h>
#include <boost/algorithm/string/join.hpp>
#include <boost/range/adaptor/map.hpp>
#include <filesystem>

View File

@ -1,5 +1,7 @@
#include <mutex>
#include <Backups/WithRetries.h>
#include <Core/Settings.h>
#include <mutex>
namespace DB
{

View File

@ -8,6 +8,7 @@
#include <Common/SensitiveDataMasker.h>
#include <Common/StringUtils.h>
#include <Common/logger_useful.h>
#include <Core/Settings.h>
#include <Formats/registerFormats.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromFile.h>

View File

@ -1,5 +1,7 @@
#include "LibraryBridgeHelper.h"
#include <Core/ServerSettings.h>
#include <Core/Settings.h>
#include <IO/ConnectionTimeouts.h>
namespace DB

View File

@ -10,6 +10,7 @@
#include <Common/Stopwatch.h>
#include <Common/DNSResolver.h>
#include <Core/ExternalTable.h>
#include <Core/Settings.h>
#include <Poco/Util/Application.h>
#include <Interpreters/Context.h>
#include <Client/Suggest.h>

View File

@ -1,6 +1,7 @@
#include <Client/ConnectionEstablisher.h>
#include <Common/quoteString.h>
#include <Common/ProfileEvents.h>
#include <Core/Settings.h>
namespace ProfileEvents
{

View File

@ -1,4 +1,5 @@
#include <Client/ConnectionPool.h>
#include <Core/Settings.h>
#include <boost/functional/hash.hpp>
@ -85,4 +86,15 @@ ConnectionPoolFactory & ConnectionPoolFactory::instance()
return ret;
}
IConnectionPool::Entry ConnectionPool::get(const DB::ConnectionTimeouts& timeouts, const DB::Settings& settings,
bool force_connected)
{
Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds());
if (force_connected)
entry->forceConnected(timeouts);
return entry;
}
}

View File

@ -4,12 +4,13 @@
#include <Common/Priority.h>
#include <Client/Connection.h>
#include <IO/ConnectionTimeouts.h>
#include <Core/Settings.h>
#include <base/defines.h>
namespace DB
{
struct Settings;
/** Interface for connection pools.
*
* Usage (using the usual `ConnectionPool` example)
@ -102,15 +103,7 @@ public:
Entry get(const ConnectionTimeouts & timeouts, /// NOLINT
const Settings & settings,
bool force_connected = true) override
{
Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds());
if (force_connected)
entry->forceConnected(timeouts);
return entry;
}
bool force_connected = true) override;
std::string getDescription() const
{

View File

@ -3,6 +3,7 @@
#include <Client/HedgedConnections.h>
#include <Common/ProfileEvents.h>
#include <Core/Settings.h>
#include <Interpreters/ClientInfo.h>
#include <Interpreters/Context.h>

View File

@ -8,13 +8,14 @@
#include <Common/Fiber.h>
#include <Client/ConnectionEstablisher.h>
#include <Client/ConnectionPoolWithFailover.h>
#include <Core/Settings.h>
#include <unordered_map>
#include <memory>
namespace DB
{
struct Settings;
/** Class for establishing hedged connections with replicas.
* The process of establishing connection is divided on stages, on each stage if
* replica doesn't respond for a long time, we start establishing connection with

View File

@ -2,6 +2,7 @@
#include <Common/thread_local_rng.h>
#include <Core/Protocol.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <IO/ConnectionTimeouts.h>
#include <IO/Operators.h>

View File

@ -2,7 +2,9 @@
#include <Interpreters/Context.h>
#include <Common/CurrentThread.h>
#include <Common/DateLUTImpl.h>
#include <Common/filesystemHelpers.h>
#include <Core/Settings.h>
#include <Poco/DigestStream.h>
#include <Poco/Exception.h>

View File

@ -1,3 +1,4 @@
#include <Common/Exception.h>
#include <Common/GetPriorityForLoadBalancing.h>
#include <Common/Priority.h>

View File

@ -1,6 +1,10 @@
#pragma once
#include <Core/SettingsEnums.h>
#include <Common/Priority.h>
#include <Core/LoadBalancing.h>
#include <functional>
#include <vector>
namespace DB
{

View File

@ -1,6 +1,7 @@
#include <Common/NamedCollections/NamedCollectionsFactory.h>
#include <Common/NamedCollections/NamedCollectionConfiguration.h>
#include <Common/NamedCollections/NamedCollectionsMetadataStorage.h>
#include <Core/Settings.h>
#include <base/sleep.h>
namespace DB

View File

@ -5,6 +5,7 @@
#include <Common/ZooKeeper/IKeeper.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/ZooKeeper/ZooKeeper.h>
#include <Core/Settings.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteHelpers.h>

View File

@ -1,6 +1,7 @@
#include "ProgressIndication.h"
#include <algorithm>
#include <cstddef>
#include <iostream>
#include <numeric>
#include <filesystem>
#include <cmath>

View File

@ -1,14 +1,15 @@
#pragma once
#include <unordered_map>
#include <unordered_set>
#include <mutex>
#include <IO/Progress.h>
#include <Interpreters/Context.h>
#include <base/types.h>
#include <Common/Stopwatch.h>
#include <Common/EventRateMeter.h>
#include <iostream>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
namespace DB
{

View File

@ -12,13 +12,14 @@
#include <base/sleep.h>
#include <IO/ReadHelpers.h>
#include <Common/logger_useful.h>
#include <Common/CurrentMemoryTracker.h>
#include <Common/Exception.h>
#include <Common/MemoryTracker.h>
#include <Common/ThreadFuzzer.h>
#include <Common/logger_useful.h>
#include <Common/thread_local_rng.h>
#include <Common/ThreadFuzzer.h>
#include "config.h" // USE_JEMALLOC

View File

@ -1,5 +1,6 @@
#include <Coordination/FourLetterCommand.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperDispatcher.h>
#include <Server/KeeperTCPHandler.h>
#include <Common/ZooKeeper/IKeeper.h>

View File

@ -3,6 +3,7 @@
#include <Coordination/KeeperContext.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/Defines.h>
#include <Coordination/KeeperConstants.h>
#include <Server/CloudPlacementInfo.h>

View File

@ -5,8 +5,8 @@
#include <Poco/Util/AbstractConfiguration.h>
#include <base/hex.h>
#include "Common/ZooKeeper/IKeeper.h"
#include "Common/ZooKeeper/ZooKeeperCommon.h"
#include <Common/ZooKeeper/IKeeper.h>
#include <Common/ZooKeeper/ZooKeeperCommon.h>
#include <Common/setThreadName.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/checkStackSize.h>
@ -14,6 +14,7 @@
#include <Common/ProfileEvents.h>
#include <Common/logger_useful.h>
#include <Common/formatReadable.h>
#include <Coordination/CoordinationSettings.h>
#include <Disks/IDisk.h>

View File

@ -11,7 +11,6 @@
#include <Common/Exception.h>
#include <functional>
#include <Coordination/KeeperServer.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/Keeper4LWInfo.h>
#include <Coordination/KeeperConnectionStats.h>
#include <Coordination/KeeperSnapshotManagerS3.h>

View File

@ -3,9 +3,7 @@
#include "config.h"
#include <chrono>
#include <mutex>
#include <string>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperLogStore.h>
#include <Coordination/KeeperSnapshotManagerS3.h>
#include <Coordination/KeeperStateMachine.h>
@ -30,6 +28,10 @@
#include <Common/getMultipleKeysFromConfig.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
#include <chrono>
#include <mutex>
#include <string>
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#include <fmt/chrono.h>

View File

@ -1,6 +1,5 @@
#pragma once
#include <Coordination/CoordinationSettings.h>
#include <Coordination/InMemoryLogStore.h>
#include <Coordination/KeeperStateMachine.h>
#include <Coordination/KeeperStateManager.h>
@ -17,6 +16,9 @@ namespace DB
using RaftAppendResult = nuraft::ptr<nuraft::cmd_result<nuraft::ptr<nuraft::buffer>>>;
struct KeeperConfigurationAndSettings;
using KeeperConfigurationAndSettingsPtr = std::shared_ptr<KeeperConfigurationAndSettings>;
class KeeperServer
{
private:

View File

@ -1,6 +1,7 @@
#include <atomic>
#include <cerrno>
#include <chrono>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperDispatcher.h>
#include <Coordination/KeeperReconfiguration.h>
#include <Coordination/KeeperSnapshotManager.h>

View File

@ -1,6 +1,5 @@
#pragma once
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperSnapshotManager.h>
#include <Coordination/KeeperSnapshotManagerS3.h>
#include <Coordination/KeeperContext.h>

View File

@ -1,14 +1,13 @@
#pragma once
#include <Core/Types.h>
#include <string>
#include <Coordination/KeeperLogStore.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperSnapshotManager.h>
#include <Core/Types.h>
#include <libnuraft/nuraft.hxx>
#include <Poco/Util/AbstractConfiguration.h>
#include "Coordination/KeeperStateMachine.h"
#include "Coordination/RaftServerConfig.h"
#include <Coordination/KeeperSnapshotManager.h>
namespace DB
{

View File

@ -20,11 +20,12 @@
#include <Common/LockMemoryExceptionInThread.h>
#include <Common/ProfileEvents.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/KeeperCommon.h>
#include <Coordination/KeeperConstants.h>
#include <Coordination/KeeperDispatcher.h>
#include <Coordination/KeeperReconfiguration.h>
#include <Coordination/KeeperStorage.h>
#include <Coordination/KeeperDispatcher.h>
#include <functional>
#include <base/defines.h>

View File

@ -7,6 +7,7 @@
#include <filesystem>
#include <thread>
#include <Coordination/Changelog.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/InMemoryLogStore.h>
#include <Coordination/KeeperContext.h>
#include <Coordination/KeeperConstants.h>

View File

@ -16,6 +16,7 @@
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <Core/ExternalTable.h>
#include <Core/Settings.h>
#include <Poco/Net/MessageHeader.h>
#include <Parsers/ASTNameTypePair.h>
#include <Parsers/ParserCreateQuery.h>

27
src/Core/LoadBalancing.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <cstdint>
namespace DB
{
enum class LoadBalancing : uint8_t
{
/// among replicas with a minimum number of errors selected randomly
RANDOM = 0,
/// a replica is selected among the replicas with the minimum number of errors
/// with the minimum number of distinguished characters in the replica name prefix and local hostname prefix
NEAREST_HOSTNAME,
/// just like NEAREST_HOSTNAME, but it count distinguished characters in a levenshtein distance manner
HOSTNAME_LEVENSHTEIN_DISTANCE,
// replicas with the same number of errors are accessed in the same order
// as they are specified in the configuration.
IN_ORDER,
/// if first replica one has higher number of errors,
/// pick a random one from replicas with minimum number of errors
FIRST_OR_RANDOM,
// round robin across replicas with the same number of errors.
ROUND_ROBIN,
};
}

View File

@ -0,0 +1,17 @@
#pragma once
#include <cstdint>
namespace DB
{
// Make it signed for compatibility with DataTypeEnum8
enum QueryLogElementType : int8_t
{
QUERY_START = 1,
QUERY_FINISH = 2,
EXCEPTION_BEFORE_START = 3,
EXCEPTION_WHILE_PROCESSING = 4,
};
}

View File

@ -0,0 +1,14 @@
#pragma once
#include <cstdint>
namespace DB
{
enum class SchemaInferenceMode : uint8_t
{
DEFAULT,
UNION,
};
}

View File

@ -1,8 +1,12 @@
#pragma once
#include <Core/Joins.h>
#include <Core/LoadBalancing.h>
#include <Core/LogsLevel.h>
#include <Core/QueryLogElementType.h>
#include <Core/SchemaInferenceMode.h>
#include <Core/SettingsFields.h>
#include <Core/ShortCircuitFunctionEvaluation.h>
#include <Formats/FormatSettings.h>
#include <IO/ReadSettings.h>
#include <Parsers/ASTSQLSecurity.h>
@ -115,25 +119,6 @@ constexpr auto getEnumValues();
return getEnumValues<EnumType>().size();\
}
enum class LoadBalancing : uint8_t
{
/// among replicas with a minimum number of errors selected randomly
RANDOM = 0,
/// a replica is selected among the replicas with the minimum number of errors
/// with the minimum number of distinguished characters in the replica name prefix and local hostname prefix
NEAREST_HOSTNAME,
/// just like NEAREST_HOSTNAME, but it count distinguished characters in a levenshtein distance manner
HOSTNAME_LEVENSHTEIN_DISTANCE,
// replicas with the same number of errors are accessed in the same order
// as they are specified in the configuration.
IN_ORDER,
/// if first replica one has higher number of errors,
/// pick a random one from replicas with minimum number of errors
FIRST_OR_RANDOM,
// round robin across replicas with the same number of errors.
ROUND_ROBIN,
};
DECLARE_SETTING_ENUM(LoadBalancing)
DECLARE_SETTING_ENUM(JoinStrictness)
@ -204,16 +189,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion)
DECLARE_SETTING_ENUM(LogsLevel)
// Make it signed for compatibility with DataTypeEnum8
enum QueryLogElementType : int8_t
{
QUERY_START = 1,
QUERY_FINISH = 2,
EXCEPTION_BEFORE_START = 3,
EXCEPTION_WHILE_PROCESSING = 4,
};
DECLARE_SETTING_ENUM_WITH_RENAME(LogQueriesType, QueryLogElementType)
@ -292,13 +267,6 @@ enum class StreamingHandleErrorMode : uint8_t
DECLARE_SETTING_ENUM(StreamingHandleErrorMode)
enum class ShortCircuitFunctionEvaluation : uint8_t
{
ENABLE, // Use short-circuit function evaluation for functions that are suitable for it.
FORCE_ENABLE, // Use short-circuit function evaluation for all functions.
DISABLE, // Disable short-circuit function evaluation.
};
DECLARE_SETTING_ENUM(ShortCircuitFunctionEvaluation)
enum class TransactionsWaitCSNMode : uint8_t
@ -367,12 +335,6 @@ DECLARE_SETTING_ENUM(ObjectStorageQueueAction)
DECLARE_SETTING_ENUM(ExternalCommandStderrReaction)
enum class SchemaInferenceMode : uint8_t
{
DEFAULT,
UNION,
};
DECLARE_SETTING_ENUM(SchemaInferenceMode)
DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeOverflowBehavior, FormatSettings::DateTimeOverflowBehavior)

View File

@ -0,0 +1,15 @@
#pragma once
#include <cstdint>
namespace DB
{
enum class ShortCircuitFunctionEvaluation : uint8_t
{
ENABLE, // Use short-circuit function evaluation for functions that are suitable for it.
FORCE_ENABLE, // Use short-circuit function evaluation for all functions.
DISABLE, // Disable short-circuit function evaluation.
};
}

View File

@ -1,15 +1,16 @@
#pragma once
#include <vector>
#include <memory>
#include <cstddef>
#include <string>
#include <Core/Field.h>
#include <Core/SettingsEnums.h>
#include <Common/IntervalKind.h>
#include <DataTypes/IDataType.h>
#include <Columns/Collator.h>
#include <cstddef>
#include <string>
#include <memory>
#include <vector>
namespace DB
{

View File

@ -4,6 +4,7 @@
#include <base/errnoToString.h>
#include <Common/CurrentThread.h>
#include <Common/MemoryTracker.h>
#include <Core/Settings.h>
#include <Daemon/BaseDaemon.h>
#include <Daemon/SentryWriter.h>
#include <Common/GWPAsan.h>

Some files were not shown because too many files have changed in this diff Show More