thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 18:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into modulo-partition-key

Browse Source
This commit is contained in:
kssenii 2021-05-24 22:08:49 +00:00
commit c3fc2ff04b
80 changed files with 1443 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmake/find/llvm.cmake
View File

@ -29,7 +29,6 @@ message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}")
# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. # This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles.
set (REQUIRED_LLVM_LIBRARIES set (REQUIRED_LLVM_LIBRARIES
LLVMOrcJIT
LLVMExecutionEngine LLVMExecutionEngine
LLVMRuntimeDyld LLVMRuntimeDyld
LLVMX86CodeGen LLVMX86CodeGen

2
contrib/libunwind vendored

@ -1 +1 @@
Subproject commit 8fe25d7dc70f2a4ea38c3e5a33fa9d4199b67a5a Subproject commit 1e4a2e5ce77be1af12e918a3c15dccf2bbac587d

2
contrib/llvm vendored

@ -1 +1 @@
Subproject commit a7198805de67374eb3fb4c6b89797fa2d1cd7e50 Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1

19
docker/test/performance-comparison/perf.py
View File

@ -44,7 +44,7 @@ parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated l
parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.') parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.')
parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.')
parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.')
parser.add_argument('--max-query-seconds', type=int, default=10, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.')
parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.') parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.')
parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.')
parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.')
@ -273,8 +273,14 @@ for query_index in queries_to_run:
prewarm_id = f'{query_prefix}.prewarm0' prewarm_id = f'{query_prefix}.prewarm0'
try: try:
# Will also detect too long queries during warmup stage # During the warmup runs, we will also:
res = c.execute(q, query_id = prewarm_id, settings = {'max_execution_time': args.max_query_seconds}) # * detect queries that are exceedingly long, to fail fast,
# * collect profiler traces, which might be helpful for analyzing
# test coverage. We disable profiler for normal runs because
# it makes the results unstable.
res = c.execute(q, query_id = prewarm_id,
settings = {'max_execution_time': args.max_query_seconds,
'query_profiler_real_time_period_ns': 10000000})
except clickhouse_driver.errors.Error as e: except clickhouse_driver.errors.Error as e:
# Add query id to the exception to make debugging easier. # Add query id to the exception to make debugging easier.
e.args = (prewarm_id, *e.args) e.args = (prewarm_id, *e.args)
@ -359,10 +365,11 @@ for query_index in queries_to_run:
# For very short queries we have a special mode where we run them for at # For very short queries we have a special mode where we run them for at
# least some time. The recommended lower bound of run time for "normal" # least some time. The recommended lower bound of run time for "normal"
# queries is about 0.1 s, and we run them about 10 times, giving the # queries is about 0.1 s, and we run them about 10 times, giving the
# time per query per server of about one second. Use this value as a # time per query per server of about one second. Run "short" queries
# reference for "short" queries. # for longer time, because they have a high percentage of overhead and
# might give less stable results.
if is_short[query_index]: if is_short[query_index]:
if server_seconds >= 2 * len(this_query_connections): if server_seconds >= 8 * len(this_query_connections):
break break
# Also limit the number of runs, so that we don't go crazy processing # Also limit the number of runs, so that we don't go crazy processing
# the results -- 'eqmed.sql' is really suboptimal. # the results -- 'eqmed.sql' is really suboptimal.

14
docker/test/performance-comparison/report.py
View File

@ -446,6 +446,9 @@ if args.report == 'main':
attrs[3] = f'style="background: {color_bad}"' attrs[3] = f'style="background: {color_bad}"'
else: else:
attrs[3] = '' attrs[3] = ''
# Just don't add the slightly unstable queries we don't consider
# errors. It's not clear what the user should do with them.
continue
text += tableRow(r, attrs, anchor) text += tableRow(r, attrs, anchor)
@ -553,12 +556,11 @@ if args.report == 'main':
error_tests += unstable_partial_queries error_tests += unstable_partial_queries
status = 'failure' status = 'failure'
if unstable_queries: # Don't show mildly unstable queries, only the very unstable ones we
message_array.append(str(unstable_queries) + ' unstable') # treat as errors.
if very_unstable_queries:
# Disabled before fix. status = 'failure'
# if very_unstable_queries: message_array.append(str(very_unstable_queries) + ' unstable')
# status = 'failure'
error_tests += slow_average_tests error_tests += slow_average_tests
if error_tests: if error_tests:

70
docs/en/operations/configuration-files.md
View File

@ -5,9 +5,9 @@ toc_title: Configuration Files
# Configuration Files {#configuration_files} # Configuration Files {#configuration_files}
ClickHouse supports multi-file configuration management. The main server configuration file is `/etc/clickhouse-server/config.xml`. Other files must be in the `/etc/clickhouse-server/config.d` directory. ClickHouse supports multi-file configuration management. The main server configuration file is `/etc/clickhouse-server/config.xml` or `/etc/clickhouse-server/config.yaml`. Other files must be in the `/etc/clickhouse-server/config.d` directory. Note, that any configuration file can be written either in XML or YAML, but mixing formats in one file is not supported. For example, you can have main configs as `config.xml` and `users.xml` and write additional files in `config.d` and `users.d` directories in `.yaml`.
All the configuration files should be in XML format. Also, they should have the same root element, usually `<yandex>`. All the configuration files should be in XML or YAML formats. All XML files should have the same root element, usually `<yandex>`. As for YAML, `yandex:` should not be present, the parser will insert it automatically.
## Override {#override} ## Override {#override}
@ -32,7 +32,7 @@ Users configuration can be splitted into separate files similar to `config.xml`
Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`. Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`.
Directory `users.d` is used by default, as `users_config` defaults to `users.xml`. Directory `users.d` is used by default, as `users_config` defaults to `users.xml`.
## Example {#example} ## XML example {#example}
For example, you can have separate config file for each user like this: For example, you can have separate config file for each user like this:
@ -55,6 +55,70 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
</yandex> </yandex>
``` ```
## YAML examples {#example}
Here you can see default config written in YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example).
There are some differences between YAML and XML formats in terms of ClickHouse configurations. Here are some tips for writing a configuration in YAML format.
You should use a Scalar node to write a key-value pair:
``` yaml
key: value
```
To create a node, containing other nodes you should use a Map:
``` yaml
map_key:
key1: val1
key2: val2
key3: val3
```
To create a list of values or nodes assigned to one tag you should use a Sequence:
``` yaml
seq_key:
- val1
- val2
- key1: val3
- map:
key2: val4
key3: val5
```
If you want to write an attribute for a Sequence or Map node, you should use a @ prefix before the attribute key. Note, that @ is reserved by YAML standard, so you should also to wrap it into double quotes:
``` yaml
map:
"@attr1": value1
"@attr2": value2
key: 123
```
From that Map we will get these XML nodes:
``` xml
<map attr1="value1" attr2="value2">
<key>123</key>
</map>
```
You can also set attributes for Sequence:
``` yaml
seq:
- "@attr1": value1
- "@attr2": value2
- 123
- abc
```
So, we can get YAML config equal to this XML one:
``` xml
<seq attr1="value1" attr2="value2">123</seq>
<seq attr1="value1" attr2="value2">abc</seq>
```
## Implementation Details {#implementation-details} ## Implementation Details {#implementation-details}
For each config file, the server also generates `file-preprocessed.xml` files when starting. These files contain all the completed substitutions and overrides, and they are intended for informational use. If ZooKeeper substitutions were used in the config files but ZooKeeper is not available on the server start, the server loads the configuration from the preprocessed file. For each config file, the server also generates `file-preprocessed.xml` files when starting. These files contain all the completed substitutions and overrides, and they are intended for informational use. If ZooKeeper substitutions were used in the config files but ZooKeeper is not available on the server start, the server loads the configuration from the preprocessed file.

3
docs/en/sql-reference/statements/select/prewhere.md
View File

@ -16,6 +16,9 @@ A query may simultaneously specify `PREWHERE` and `WHERE`. In this case, `PREWHE
If the `optimize_move_to_prewhere` setting is set to 0, heuristics to automatically move parts of expressions from `WHERE` to `PREWHERE` are disabled. If the `optimize_move_to_prewhere` setting is set to 0, heuristics to automatically move parts of expressions from `WHERE` to `PREWHERE` are disabled.
!!! note "Attention"
The `PREWHERE` section is executed before` FINAL`, so the results of `FROM FINAL` queries may be skewed when using` PREWHERE` with fields not in the `ORDER BY` section of a table.
## Limitations {#limitations} ## Limitations {#limitations}
`PREWHERE` is only supported by tables from the `*MergeTree` family. `PREWHERE` is only supported by tables from the `*MergeTree` family.

69
docs/ru/operations/configuration-files.md
View File

@ -6,9 +6,9 @@ toc_title: "Конфигурационные файлы"
# Конфигурационные файлы {#configuration_files} # Конфигурационные файлы {#configuration_files}
Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. Основной конфигурационный файл сервера - `config.xml` или `config.yaml`. Он расположен в директории `/etc/clickhouse-server/`.
Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf` из директории `config.d` рядом с конфигом. Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf`, а также `.yaml` (для файлов в формате YAML) из директории `config.d` рядом с конфигом.
У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`. У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`.
@ -25,7 +25,7 @@ toc_title: "Конфигурационные файлы"
В элементе `users_config` файла `config.xml` можно указать относительный путь к конфигурационному файлу с настройками пользователей, профилей и квот. Значение `users_config` по умолчанию — `users.xml`. Если `users_config` не указан, то настройки пользователей, профилей и квот можно задать непосредственно в `config.xml`. В элементе `users_config` файла `config.xml` можно указать относительный путь к конфигурационному файлу с настройками пользователей, профилей и квот. Значение `users_config` по умолчанию — `users.xml`. Если `users_config` не указан, то настройки пользователей, профилей и квот можно задать непосредственно в `config.xml`.
Настройки пользователя могут быть разделены в несколько отдельных файлов аналогичных `config.xml` и `config.d\`. Имя директории задаётся также как `users_config`. Настройки пользователя могут быть разделены в несколько отдельных файлов аналогичных `config.xml` и `config.d\`. Имя директории задаётся также как `users_config`.
Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`. Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`/`.yaml`.
Директория `users.d` используется по умолчанию, также как `users.xml` используется для `users_config`. Директория `users.d` используется по умолчанию, также как `users.xml` используется для `users_config`.
Например, можно иметь по отдельному конфигурационному файлу для каждого пользователя: Например, можно иметь по отдельному конфигурационному файлу для каждого пользователя:
@ -52,3 +52,66 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера. Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера.
## Примеры записи конфигурации на YAML {#example}
Здесь можно рассмотреть пример реальной конфигурации записанной на YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example).
Между стандартами XML и YAML имеются различия, поэтому в этом разделе будут перечислены некоторые подсказки для написания конфигурации на YMAL.
Для записи обычной пары ключ-значение следует использовать Scalar:
``` yaml
key: value
```
Для создания тега, содержащего подтеги следует использовать Map:
``` yaml
map_key:
key1: val1
key2: val2
key3: val3
```
Для создания списка значений или подтегов, расположенных по определенному ключу, следует использовать Sequence:
``` yaml
seq_key:
- val1
- val2
- key1: val3
- map:
key2: val4
key3: val5
```
В случае, усли необходимо объявить тег, аналогичный XML-атрибуту, необходимо задать скаляр, имеющий ключ с префиксом @ и заключенный в кавычки:
``` yaml
map:
"@attr1": value1
"@attr2": value2
key: 123
```
Из такой Map мы получим после конвертации:
``` xml
<map attr1="value1" attr2="value2">
<key>123</key>
</map>
```
Помимо Map, можно задавать атрибуты для Sequence:
``` yaml
seq:
- "@attr1": value1
- "@attr2": value2
- 123
- abc
```
Таким образом получая аналог следующей записи на XML:
``` xml
<seq attr1="value1" attr2="value2">123</seq>
<seq attr1="value1" attr2="value2">abc</seq>
```

5
docs/ru/sql-reference/dictionaries/index.md
View File

@ -12,6 +12,5 @@ ClickHouse поддерживает специальные функции для
ClickHouse поддерживает: ClickHouse поддерживает:
- [Встроенные словари](internal-dicts.md#internal_dicts) со специфическим [набором функций](../../sql-reference/dictionaries/ext-dict-functions/). - [Встроенные словари](internal-dicts.md#internal_dicts) со специфическим [набором функций](../../sql-reference/functions/ext-dict-functions.md).
- [Подключаемые (внешние) словари](external-dictionaries/external-dicts.md#dicts-external-dicts) с [набором функций](../../sql-reference/dictionaries/ext-dict-functions/). - [Подключаемые (внешние) словари](external-dictionaries/external-dicts.md#dicts-external-dicts) с [набором функций](../../sql-reference/functions/ext-dict-functions.md).

3
docs/ru/sql-reference/statements/select/prewhere.md
View File

@ -16,6 +16,9 @@ Prewhere — это оптимизация для более эффективн
Если значение параметра `optimize_move_to_prewhere` равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается. Если значение параметра `optimize_move_to_prewhere` равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается.
!!! note "Внимание"
Секция `PREWHERE` выполняется до `FINAL`, поэтому результаты запросов `FROM FINAL` могут исказится при использовании `PREWHERE` с полями не входящями в `ORDER BY` таблицы.
## Ограничения {#limitations} ## Ограничения {#limitations}
`PREWHERE` поддерживается только табличными движками из семейства `*MergeTree`. `PREWHERE` поддерживается только табличными движками из семейства `*MergeTree`.

86
programs/server/config-example.yaml.disabled
View File

@ -1,86 +0,0 @@
# We can use 3 main node types in YAML: Scalar, Map and Sequence.
# A Scalar is a simple key-value pair:
scalar: 123
# Here we have a key "scalar" and value "123"
# If we rewrite this in XML, we will get <scalar>123</scalar>
# We can also represent an empty value with '':
key: ''
# A Map is a node, which contains other nodes:
map:
key1: value1
key2: value2
small_map:
key3: value3
# This map can be converted into:
# <map>
# <key1>value1</key1>
# <key2>value2</key2>
# <small_map>
# <key3>value3</key3>
# </small_map>
# </map>
# A Sequence is a node, which contains also other nodes.
# The main difference from Map is that Sequence can also contain simple values.
sequence:
- val1
- val2
- key: 123
- map:
mkey1: foo
mkey2: bar
# We can represent it in XML this way:
# <sequence>val1</sequence>
# <sequence>val2</sequence>
# <sequence>
# <key>123</key>
# </sequence>
# <sequence>
# <map>
# <mkey1>foo</mkey1>
# <mkey2>bar</mkey2>
# </map>
# </sequence>
# YAML does not have direct support for structures like XML attributes.
# We represent them as nodes with @ prefix in key. Note, that @ is reserved by YAML standard,
# so you will need to write double quotes around the key. Both Map and Sequence can have
# attributes as children nodes
map:
"@attr1": value1
"@attr2": value2
key: 123
# This gives us:
# <map attr1="value1" attr2="value2">
# <key>123</key>
# </map>
sequence:
- "@attr1": value1
- "@attr2": value2
- 123
- abc
# And this gives us:
# <map attr1="value1" attr2="value2">123</map>
# <map attr1="value1" attr2="value2">abc</map>

950
programs/server/config.yaml.example Normal file
View File

@ -0,0 +1,950 @@
# This is an example of a configuration file "config.xml" rewritten in YAML
# You can read this documentation for detailed information about YAML configuration:
# https://clickhouse.tech/docs/en/operations/configuration-files/
# NOTE: User and query level settings are set up in "users.yaml" file.
# If you have accidentally specified user-level settings here, server won't start.
# You can either move the settings to the right place inside "users.xml" file
# or add skip_check_for_incorrect_settings: 1 here.
logger:
# Possible levels [1]:
# - none (turns off logging)
# - fatal
# - critical
# - error
# - warning
# - notice
# - information
# - debug
# - trace
# [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
level: trace
log: /var/log/clickhouse-server/clickhouse-server.log
errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
# Rotation policy
# See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
size: 1000M
count: 10
# console: 1
# Default behavior is autodetection (log to console if not daemon mode and is tty)
# Per level overrides (legacy):
# For example to suppress logging of the ConfigReloader you can use:
# NOTE: levels.logger is reserved, see below.
# levels:
# ConfigReloader: none
# Per level overrides:
# For example to suppress logging of the RBAC for default user you can use:
# (But please note that the logger name maybe changed from version to version, even after minor upgrade)
# levels:
# - logger:
# name: 'ContextAccess (default)'
# level: none
# - logger:
# name: 'DatabaseOrdinary (test)'
# level: none
# It is the name that will be shown in the clickhouse-client.
# By default, anything with "production" will be highlighted in red in query prompt.
# display_name: production
# Port for HTTP API. See also 'https_port' for secure connections.
# This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...)
# and by most of web interfaces (embedded UI, Grafana, Redash, ...).
http_port: 8123
# Port for interaction by native protocol with:
# - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark, clickhouse-copier);
# - clickhouse-server with other clickhouse-servers for distributed query processing;
# - ClickHouse drivers and applications supporting native protocol
# (this protocol is also informally called as "the TCP protocol");
# See also 'tcp_port_secure' for secure connections.
tcp_port: 9000
# Compatibility with MySQL protocol.
# ClickHouse will pretend to be MySQL for applications connecting to this port.
mysql_port: 9004
# Compatibility with PostgreSQL protocol.
# ClickHouse will pretend to be PostgreSQL for applications connecting to this port.
postgresql_port: 9005
# HTTP API with TLS (HTTPS).
# You have to configure certificate to enable this interface.
# See the openSSL section below.
# https_port: 8443
# Native interface with TLS.
# You have to configure certificate to enable this interface.
# See the openSSL section below.
# tcp_port_secure: 9440
# Native interface wrapped with PROXYv1 protocol
# PROXYv1 header sent for every connection.
# ClickHouse will extract information about proxy-forwarded client address from the header.
# tcp_with_proxy_port: 9011
# Port for communication between replicas. Used for data exchange.
# It provides low-level data access between servers.
# This port should not be accessible from untrusted networks.
# See also 'interserver_http_credentials'.
# Data transferred over connections to this port should not go through untrusted networks.
# See also 'interserver_https_port'.
interserver_http_port: 9009
# Port for communication between replicas with TLS.
# You have to configure certificate to enable this interface.
# See the openSSL section below.
# See also 'interserver_http_credentials'.
# interserver_https_port: 9010
# Hostname that is used by other replicas to request this server.
# If not specified, than it is determined analogous to 'hostname -f' command.
# This setting could be used to switch replication to another network interface
# (the server may be connected to multiple networks via multiple addresses)
# interserver_http_host: example.yandex.ru
# You can specify credentials for authenthication between replicas.
# This is required when interserver_https_port is accessible from untrusted networks,
# and also recommended to avoid SSRF attacks from possibly compromised services in your network.
# interserver_http_credentials:
# user: interserver
# password: ''
# Listen specified address.
# Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere.
# Notes:
# If you open connections from wildcard address, make sure that at least one of the following measures applied:
# - server is protected by firewall and not accessible from untrusted networks;
# - all users are restricted to subset of network addresses (see users.xml);
# - all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces.
# - users without password have readonly access.
# See also: https://www.shodan.io/search?query=clickhouse
# listen_host: '::'
# Same for hosts without support for IPv6:
# listen_host: 0.0.0.0
# Default values - try listen localhost on IPv4 and IPv6.
# listen_host: '::1'
# listen_host: 127.0.0.1
# Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen.
# listen_try: 0
# Allow multiple servers to listen on the same address:port. This is not recommended.
# listen_reuse_port: 0
# listen_backlog: 64
max_connections: 4096
# For 'Connection: keep-alive' in HTTP 1.1
keep_alive_timeout: 3
# gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API)
# grpc_port: 9100
grpc:
enable_ssl: false
# The following two files are used only if enable_ssl=1
ssl_cert_file: /path/to/ssl_cert_file
ssl_key_file: /path/to/ssl_key_file
# Whether server will request client for a certificate
ssl_require_client_auth: false
# The following file is used only if ssl_require_client_auth=1
ssl_ca_cert_file: /path/to/ssl_ca_cert_file
# Default compression algorithm (applied if client doesn't specify another algorithm).
# Supported algorithms: none, deflate, gzip, stream_gzip
compression: deflate
# Default compression level (applied if client doesn't specify another level).
# Supported levels: none, low, medium, high
compression_level: medium
# Send/receive message size limits in bytes. -1 means unlimited
max_send_message_size: -1
max_receive_message_size: -1
# Enable if you want very detailed logs
verbose_logs: false
# Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71
openSSL:
server:
# Used for https server AND secure tcp port
# openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
certificateFile: /etc/clickhouse-server/server.crt
privateKeyFile: /etc/clickhouse-server/server.key
# dhparams are optional. You can delete the dhParamsFile: element.
# To generate dhparams, use the following command:
# openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
# Only file format with BEGIN DH PARAMETERS is supported.
dhParamsFile: /etc/clickhouse-server/dhparam.pem
verificationMode: none
loadDefaultCAFile: true
cacheSessions: true
disableProtocols: 'sslv2,sslv3'
preferServerCiphers: true
client:
# Used for connecting to https dictionary source and secured Zookeeper communication
loadDefaultCAFile: true
cacheSessions: true
disableProtocols: 'sslv2,sslv3'
preferServerCiphers: true
# Use for self-signed: verificationMode: none
invalidCertificateHandler:
# Use for self-signed: name: AcceptCertificateHandler
name: RejectCertificateHandler
# Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123
# http_server_default_response: |-
# <html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>
# Maximum number of concurrent queries.
max_concurrent_queries: 100
# Maximum memory usage (resident set size) for server process.
# Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM.
# If the value is larger than "max_server_memory_usage_to_ram_ratio" of available physical RAM, it will be cut down.
# The constraint is checked on query execution time.
# If a query tries to allocate memory and the current memory usage plus allocation is greater
# than specified threshold, exception will be thrown.
# It is not practical to set this constraint to small values like just a few gigabytes,
# because memory allocator will keep this amount of memory in caches and the server will deny service of queries.
max_server_memory_usage: 0
# Maximum number of threads in the Global thread pool.
# This will default to a maximum of 10000 threads if not specified.
# This setting will be useful in scenarios where there are a large number
# of distributed queries that are running concurrently but are idling most
# of the time, in which case a higher number of threads might be required.
max_thread_pool_size: 10000
# On memory constrained environments you may have to set this to value larger than 1.
max_server_memory_usage_to_ram_ratio: 0.9
# Simple server-wide memory profiler. Collect a stack trace at every peak allocation step (in bytes).
# Data will be stored in system.trace_log table with query_id = empty string.
# Zero means disabled.
total_memory_profiler_step: 4194304
# Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type.
# The probability is for every alloc/free regardless to the size of the allocation.
# Note that sampling happens only when the amount of untracked memory exceeds the untracked memory limit,
# which is 4 MiB by default but can be lowered if 'total_memory_profiler_step' is lowered.
# You may want to set 'total_memory_profiler_step' to 1 for extra fine grained sampling.
total_memory_tracker_sample_probability: 0
# Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
# correct maximum value.
# max_open_files: 262144
# Size of cache of uncompressed blocks of data, used in tables of MergeTree family.
# In bytes. Cache is single for server. Memory is allocated only on demand.
# Cache is used when 'use_uncompressed_cache' user setting turned on (off by default).
# Uncompressed cache is advantageous only for very short queries and in rare cases.
# Note: uncompressed cache can be pointless for lz4, because memory bandwidth
# is slower than multi-core decompression on some server configurations.
# Enabling it can sometimes paradoxically make queries slower.
uncompressed_cache_size: 8589934592
# Approximate size of mark cache, used in tables of MergeTree family.
# In bytes. Cache is single for server. Memory is allocated only on demand.
# You should not lower this value.
mark_cache_size: 5368709120
# If you enable the `min_bytes_to_use_mmap_io` setting,
# the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace.
# It makes sense only for large files and helps only if data reside in page cache.
# To avoid frequent open/mmap/munmap/close calls (which are very expensive due to consequent page faults)
# and to reuse mappings from several threads and queries,
# the cache of mapped files is maintained. Its size is the number of mapped regions (usually equal to the number of mapped files).
# The amount of data in mapped files can be monitored
# in system.metrics, system.metric_log by the MMappedFiles, MMappedFileBytes metrics
# and in system.asynchronous_metrics, system.asynchronous_metrics_log by the MMapCacheCells metric,
# and also in system.events, system.processes, system.query_log, system.query_thread_log by the
# CreatedReadBufferMMap, CreatedReadBufferMMapFailed, MMappedFileCacheHits, MMappedFileCacheMisses events.
# Note that the amount of data in mapped files does not consume memory directly and is not accounted
# in query or server memory usage - because this memory can be discarded similar to OS page cache.
# The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree,
# also it can be dropped manually by the SYSTEM DROP MMAP CACHE query.
mmap_cache_size: 1000
# Cache size for compiled expressions.
compiled_expression_cache_size: 1073741824
# Path to data directory, with trailing slash.
path: /var/lib/clickhouse/
# Path to temporary data for processing hard queries.
tmp_path: /var/lib/clickhouse/tmp/
# Policy from the <storage_configuration> for the temporary files.
# If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
# Notes:
# - move_factor is ignored
# - keep_free_space_bytes is ignored
# - max_data_part_size_bytes is ignored
# - you must have exactly one volume in that policy
# tmp_policy: tmp
# Directory with user provided files that are accessible by 'file' table function.
user_files_path: /var/lib/clickhouse/user_files/
# LDAP server definitions.
ldap_servers: ''
# List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users,
# who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories.
# Parameters:
# host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty.
# port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise.
# bind_dn - template used to construct the DN to bind to.
# The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual
# user name during each authentication attempt.
# user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user.
# This is mainly used in search filters for further role mapping when the server is Active Directory. The
# resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default,
# user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected
# user DN value.
# base_dn - template used to construct the base DN for the LDAP search.
# The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
# of the template with the actual user name and bind DN during the LDAP search.
# scope - scope of the LDAP search.
# Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
# search_filter - template used to construct the search filter for the LDAP search.
# The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
# substrings of the template with the actual user name, bind DN, and base DN during the LDAP search.
# Note, that the special characters must be escaped properly in XML.
# verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
# to be successfully authenticated for all consecutive requests without contacting the LDAP server.
# Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
# enable_tls - flag to trigger use of secure connection to the LDAP server.
# Specify 'no' for plain text (ldap://) protocol (not recommended).
# Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default).
# Specify 'starttls' for legacy StartTLS protocol (plain text (ldap://) protocol, upgraded to TLS).
# tls_minimum_protocol_version - the minimum protocol version of SSL/TLS.
# Accepted values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2' (the default).
# tls_require_cert - SSL/TLS peer certificate verification behavior.
# Accepted values are: 'never', 'allow', 'try', 'demand' (the default).
# tls_cert_file - path to certificate file.
# tls_key_file - path to certificate key file.
# tls_ca_cert_file - path to CA certificate file.
# tls_ca_cert_dir - path to the directory containing CA certificates.
# tls_cipher_suite - allowed cipher suite (in OpenSSL notation).
# Example:
# my_ldap_server:
# host: localhost
# port: 636
# bind_dn: 'uid={user_name},ou=users,dc=example,dc=com'
# verification_cooldown: 300
# enable_tls: yes
# tls_minimum_protocol_version: tls1.2
# tls_require_cert: demand
# tls_cert_file: /path/to/tls_cert_file
# tls_key_file: /path/to/tls_key_file
# tls_ca_cert_file: /path/to/tls_ca_cert_file
# tls_ca_cert_dir: /path/to/tls_ca_cert_dir
# tls_cipher_suite: ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384
# Example (typical Active Directory with configured user DN detection for further role mapping):
# my_ad_server:
# host: localhost
# port: 389
# bind_dn: 'EXAMPLE\{user_name}'
# user_dn_detection:
# base_dn: CN=Users,DC=example,DC=com
# search_filter: '(&amp;(objectClass=user)(sAMAccountName={user_name}))'
# enable_tls: no
# To enable Kerberos authentication support for HTTP requests (GSS-SPNEGO), for those users who are explicitly configured
# to authenticate via Kerberos, define a single 'kerberos' section here.
# Parameters:
# principal - canonical service principal name, that will be acquired and used when accepting security contexts.
# This parameter is optional, if omitted, the default principal will be used.
# This parameter cannot be specified together with 'realm' parameter.
# realm - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it.
# This parameter is optional, if omitted, no additional filtering by realm will be applied.
# This parameter cannot be specified together with 'principal' parameter.
# Example:
# kerberos: ''
# Example:
# kerberos:
# principal: HTTP/clickhouse.example.com@EXAMPLE.COM
# Example:
# kerberos:
# realm: EXAMPLE.COM
# Sources to read users, roles, access rights, profiles of settings, quotas.
user_directories:
users_xml:
# Path to configuration file with predefined users.
path: users.yaml
local_directory:
# Path to folder where users created by SQL commands are stored.
path: /var/lib/clickhouse/access/
# # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
# # with the following parameters:
# # server - one of LDAP server names defined in 'ldap_servers' config section above.
# # This parameter is mandatory and cannot be empty.
# # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server.
# # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any
# # actions after authentication.
# # role_mapping - section with LDAP search parameters and mapping rules.
# # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the
# # name of the logged in user. For each entry found during that search, the value of the specified attribute is
# # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the
# # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by
# # CREATE ROLE command.
# # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be
# # applied.
# # base_dn - template used to construct the base DN for the LDAP search.
# # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}'
# # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search.
# # scope - scope of the LDAP search.
# # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
# # search_filter - template used to construct the search filter for the LDAP search.
# # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and
# # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during
# # each LDAP search.
# # Note, that the special characters must be escaped properly in XML.
# # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default.
# # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by
# # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated
# # as local role names. Empty, by default.
# # Example:
# # ldap:
# # server: my_ldap_server
# # roles:
# # my_local_role1: ''
# # my_local_role2: ''
# # role_mapping:
# # base_dn: 'ou=groups,dc=example,dc=com'
# # scope: subtree
# # search_filter: '(&amp;(objectClass=groupOfNames)(member={bind_dn}))'
# # attribute: cn
# # prefix: clickhouse_
# # Example (typical Active Directory with role mapping that relies on the detected user DN):
# # ldap:
# # server: my_ad_server
# # role_mapping:
# # base_dn: 'CN=Users,DC=example,DC=com'
# # attribute: CN
# # scope: subtree
# # search_filter: '(&amp;(objectClass=group)(member={user_dn}))'
# # prefix: clickhouse_
# Default profile of settings.
default_profile: default
# Comma-separated list of prefixes for user-defined settings.
# custom_settings_prefixes: ''
# System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on).
# system_profile: default
# Buffer profile of settings.
# This settings are used by Buffer storage to flush data to the underlying table.
# Default: used from system_profile directive.
# buffer_profile: default
# Default database.
default_database: default
# Server time zone could be set here.
# Time zone is used when converting between String and DateTime types,
# when printing DateTime in text formats and parsing DateTime from text,
# it is used in date and time related functions, if specific time zone was not passed as an argument.
# Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
# If not specified, system time zone at server startup is used.
# Please note, that server could display time zone alias instead of specified name.
# Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
# timezone: Europe/Moscow
# You can specify umask here (see "man umask"). Server will apply it on startup.
# Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
# umask: 022
# Perform mlockall after startup to lower first queries latency
# and to prevent clickhouse executable from being paged out under high IO load.
# Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
mlock_executable: true
# Reallocate memory for machine code ("text") using huge pages. Highly experimental.
remap_executable: false
# Uncomment below in order to use JDBC table engine and function.
# To install and run JDBC bridge in background:
# * [Debian/Ubuntu]
# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
# apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
# clickhouse-jdbc-bridge &
# * [CentOS/RHEL]
# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
# yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
# clickhouse-jdbc-bridge &
# Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
# jdbc_bridge:
# host: 127.0.0.1
# port: 9019
# Configuration of clusters that could be used in Distributed tables.
# https://clickhouse.tech/docs/en/operations/table_engines/distributed/
remote_servers:
# Test only shard config for testing distributed storage
test_shard_localhost:
# Inter-server per-cluster secret for Distributed queries
# default: no secret (no authentication will be performed)
# If set, then Distributed queries will be validated on shards, so at least:
# - such cluster should exist on the shard,
# - such cluster should have the same secret.
# And also (and which is more important), the initial_user will
# be used as current user for the query.
# Right now the protocol is pretty simple and it only takes into account:
# - cluster name
# - query
# Also it will be nice if the following will be implemented:
# - source hostname (see interserver_http_host), but then it will depends from DNS,
# it can use IP address instead, but then the you need to get correct on the initiator node.
# - target hostname / ip address (same notes as for source hostname)
# - time-based security tokens
# secret: ''
shard:
# Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas).
# internal_replication: false
# Optional. Shard weight when writing data. Default: 1.
# weight: 1
replica:
host: localhost
port: 9000
# Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority).
# priority: 1
test_cluster_two_shards_localhost:
shard:
- replica:
host: localhost
port: 9000
- replica:
host: localhost
port: 9000
test_cluster_two_shards:
shard:
- replica:
host: 127.0.0.1
port: 9000
- replica:
host: 127.0.0.2
port: 9000
test_cluster_two_shards_internal_replication:
shard:
- internal_replication: true
replica:
host: 127.0.0.1
port: 9000
- internal_replication: true
replica:
host: 127.0.0.2
port: 9000
test_shard_localhost_secure:
shard:
replica:
host: localhost
port: 9440
secure: 1
test_unavailable_shard:
shard:
- replica:
host: localhost
port: 9000
- replica:
host: localhost
port: 1
# The list of hosts allowed to use in URL-related storage engines and table functions.
# If this section is not present in configuration, all hosts are allowed.
# remote_url_allow_hosts:
# Host should be specified exactly as in URL. The name is checked before DNS resolution.
# Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts.
# If port is explicitly specified in URL, the host:port is checked as a whole.
# If host specified here without port, any port with this host allowed.
# "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed.
# If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]".
# If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked.
# Regular expression can be specified. RE2 engine is used for regexps.
# Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter
# (forgetting to do so is a common source of error).
# If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file.
# By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element.
# Values for substitutions are specified in /yandex/name_of_substitution elements in that file.
# ZooKeeper is used to store metadata about replicas, when using Replicated tables.
# Optional. If you don't use replicated tables, you could omit that.
# See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/
# zookeeper:
# - node:
# host: example1
# port: 2181
# - node:
# host: example2
# port: 2181
# - node:
# host: example3
# port: 2181
# Substitutions for parameters of replicated tables.
# Optional. If you don't use replicated tables, you could omit that.
# See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables
# macros:
# shard: 01
# replica: example01-01-1
# Reloading interval for embedded dictionaries, in seconds. Default: 3600.
builtin_dictionaries_reload_interval: 3600
# Maximum session timeout, in seconds. Default: 3600.
max_session_timeout: 3600
# Default session timeout, in seconds. Default: 60.
default_session_timeout: 60
# Sending data to Graphite for monitoring. Several sections can be defined.
# interval - send every X second
# root_path - prefix for keys
# hostname_in_path - append hostname to root_path (default = true)
# metrics - send data from table system.metrics
# events - send data from table system.events
# asynchronous_metrics - send data from table system.asynchronous_metrics
# graphite:
# host: localhost
# port: 42000
# timeout: 0.1
# interval: 60
# root_path: one_min
# hostname_in_path: true
# metrics: true
# events: true
# events_cumulative: false
# asynchronous_metrics: true
# graphite:
# host: localhost
# port: 42000
# timeout: 0.1
# interval: 1
# root_path: one_sec
# metrics: true
# events: true
# events_cumulative: false
# asynchronous_metrics: false
# Serve endpoint for Prometheus monitoring.
# endpoint - mertics path (relative to root, statring with "/")
# port - port to setup server. If not defined or 0 than http_port used
# metrics - send data from table system.metrics
# events - send data from table system.events
# asynchronous_metrics - send data from table system.asynchronous_metrics
# status_info - send data from different component from CH, ex: Dictionaries status
# prometheus:
# endpoint: /metrics
# port: 9363
# metrics: true
# events: true
# asynchronous_metrics: true
# status_info: true
# Query log. Used only for queries with setting log_queries = 1.
query_log:
# What table to insert data. If table is not exist, it will be created.
# When query log structure is changed after system update,
# then old table will be renamed and new table will be created automatically.
database: system
table: query_log
# PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/mergetree-family/custom_partitioning_key/
# Example:
# event_date
# toMonday(event_date)
# toYYYYMM(event_date)
# toStartOfHour(event_time)
partition_by: toYYYYMM(event_date)
# Table TTL specification: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl
# Example:
# event_date + INTERVAL 1 WEEK
# event_date + INTERVAL 7 DAY DELETE
# event_date + INTERVAL 2 WEEK TO DISK 'bbb'
# ttl: 'event_date + INTERVAL 30 DAY DELETE'
# Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters,
# Example: engine: 'ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
# Interval of flushing data.
flush_interval_milliseconds: 7500
# Trace log. Stores stack traces collected by query profilers.
# See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings.
trace_log:
database: system
table: trace_log
partition_by: toYYYYMM(event_date)
flush_interval_milliseconds: 7500
# Query thread log. Has information about all threads participated in query execution.
# Used only for queries with setting log_query_threads = 1.
query_thread_log:
database: system
table: query_thread_log
partition_by: toYYYYMM(event_date)
flush_interval_milliseconds: 7500
# Uncomment if use part log.
# Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).
# part_log:
# database: system
# table: part_log
# flush_interval_milliseconds: 7500
# Uncomment to write text log into table.
# Text log contains all information from usual server log but stores it in structured and efficient way.
# The level of the messages that goes to the table can be limited (<level>), if not specified all messages will go to the table.
# text_log:
# database: system
# table: text_log
# flush_interval_milliseconds: 7500
# level: ''
# Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval.
metric_log:
database: system
table: metric_log
flush_interval_milliseconds: 7500
collect_interval_milliseconds: 1000
# Asynchronous metric log contains values of metrics from
# system.asynchronous_metrics.
asynchronous_metric_log:
database: system
table: asynchronous_metric_log
# Asynchronous metrics are updated once a minute, so there is
# no need to flush more often.
flush_interval_milliseconds: 60000
# OpenTelemetry log contains OpenTelemetry trace spans.
opentelemetry_span_log:
# The default table creation code is insufficient, this <engine> spec
# is a workaround. There is no 'event_time' for this log, but two times,
# start and finish. It is sorted by finish time, to avoid inserting
# data too far away in the past (probably we can sometimes insert a span
# that is seconds earlier than the last span in the table, due to a race
# between several spans inserted in parallel). This gives the spans a
# global order that we can use to e.g. retry insertion into some external
# system.
engine: |-
engine MergeTree
partition by toYYYYMM(finish_date)
order by (finish_date, finish_time_us, trace_id)
database: system
table: opentelemetry_span_log
flush_interval_milliseconds: 7500
# Crash log. Stores stack traces for fatal errors.
# This table is normally empty.
crash_log:
database: system
table: crash_log
partition_by: ''
flush_interval_milliseconds: 1000
# Parameters for embedded dictionaries, used in Yandex.Metrica.
# See https://clickhouse.yandex/docs/en/dicts/internal_dicts/
# Path to file with region hierarchy.
# path_to_regions_hierarchy_file: /opt/geo/regions_hierarchy.txt
# Path to directory with files containing names of regions
# path_to_regions_names_files: /opt/geo/
# top_level_domains_path: /var/lib/clickhouse/top_level_domains/
# Custom TLD lists.
# Format: name: /path/to/file
# Changes will not be applied w/o server restart.
# Path to the list is under top_level_domains_path (see above).
top_level_domains_lists: ''
# public_suffix_list: /path/to/public_suffix_list.dat
# Configuration of external dictionaries. See:
# https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts
dictionaries_config: '*_dictionary.xml'
# Uncomment if you want data to be compressed 30-100% better.
# Don't do that if you just started using ClickHouse.
# compression:
# # Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used.
# case:
# Conditions. All must be satisfied. Some conditions may be omitted.
# # min_part_size: 10000000000 # Min part size in bytes.
# # min_part_size_ratio: 0.01 # Min size of part relative to whole table size.
# # What compression method to use.
# method: zstd
# Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
# Works only if ZooKeeper is enabled. Comment it if such functionality isn't required.
distributed_ddl:
# Path in ZooKeeper to queue with DDL queries
path: /clickhouse/task_queue/ddl
# Settings from this profile will be used to execute DDL queries
# profile: default
# Controls how much ON CLUSTER queries can be run simultaneously.
# pool_size: 1
# Cleanup settings (active tasks will not be removed)
# Controls task TTL (default 1 week)
# task_max_lifetime: 604800
# Controls how often cleanup should be performed (in seconds)
# cleanup_delay_period: 60
# Controls how many tasks could be in the queue
# max_tasks_in_queue: 1000
# Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h
# merge_tree:
# max_suspicious_broken_parts: 5
# Protection from accidental DROP.
# If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
# If you want do delete one table and don't want to change clickhouse-server config, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
# By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
# The same for max_partition_size_to_drop.
# Uncomment to disable protection.
# max_table_size_to_drop: 0
# max_partition_size_to_drop: 0
# Example of parameters for GraphiteMergeTree table engine
graphite_rollup_example:
pattern:
regexp: click_cost
function: any
retention:
- age: 0
precision: 3600
- age: 86400
precision: 60
default:
function: max
retention:
- age: 0
precision: 60
- age: 3600
precision: 300
- age: 86400
precision: 3600
# Directory in <clickhouse-path> containing schema files for various input formats.
# The directory will be created if it doesn't exist.
format_schema_path: /var/lib/clickhouse/format_schemas/
# Default query masking rules, matching lines would be replaced with something else in the logs
# (both text logs and system.query_log).
# name - name for the rule (optional)
# regexp - RE2 compatible regular expression (mandatory)
# replace - substitution string for sensitive data (optional, by default - six asterisks)
query_masking_rules:
rule:
name: hide encrypt/decrypt arguments
regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)'
# or more secure, but also more invasive:
# (aes_\w+)\s*\(.*\)
replace: \1(???)
# Uncomment to use custom http handlers.
# rules are checked from top to bottom, first match runs the handler
# url - to match request URL, you can use 'regex:' prefix to use regex match(optional)
# methods - to match request method, you can use commas to separate multiple method matches(optional)
# headers - to match request headers, match each child element(child element name is header name), you can use 'regex:' prefix to use regex match(optional)
# handler is request handler
# type - supported types: static, dynamic_query_handler, predefined_query_handler
# query - use with predefined_query_handler type, executes query when the handler is called
# query_param_name - use with dynamic_query_handler type, extracts and executes the value corresponding to the <query_param_name> value in HTTP request params
# status - use with static type, response status code
# content_type - use with static type, response content-type
# response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client.
# http_handlers:
# - rule:
# url: /
# methods: POST,GET
# headers:
# pragma: no-cache
# handler:
# type: dynamic_query_handler
# query_param_name: query
# - rule:
# url: /predefined_query
# methods: POST,GET
# handler:
# type: predefined_query_handler
# query: 'SELECT * FROM system.settings'
# - rule:
# handler:
# type: static
# status: 200
# content_type: 'text/plain; charset=UTF-8'
# response_content: config://http_server_default_response
send_crash_reports:
# Changing <enabled> to true allows sending crash reports to
# the ClickHouse core developers team via Sentry https://sentry.io
# Doing so at least in pre-production environments is highly appreciated
enabled: false
# Change <anonymize> to true if you don't feel comfortable attaching the server hostname to the crash report
anonymize: false
# Default endpoint should be changed to different Sentry DSN only if you have
# some in-house engineers or hired consultants who're going to debug ClickHouse issues for you
endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
# Uncomment to disable ClickHouse internal DNS caching.
# disable_internal_dns_cache: 1

107
programs/server/users.yaml.example Normal file
View File

@ -0,0 +1,107 @@
# Profiles of settings.
profiles:
# Default settings.
default:
# Maximum memory usage for processing single query, in bytes.
max_memory_usage: 10000000000
# How to choose between replicas during distributed query processing.
# random - choose random replica from set of replicas with minimum number of errors
# nearest_hostname - from set of replicas with minimum number of errors, choose replica
# with minimum number of different symbols between replica's hostname and local hostname (Hamming distance).
# in_order - first live replica is chosen in specified order.
# first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
load_balancing: random
# Profile that allows only read queries.
readonly:
readonly: 1
# Users and ACL.
users:
# If user name was not specified, 'default' user is used.
default:
# Password could be specified in plaintext or in SHA256 (in hex format).
#
# If you want to specify password in plaintext (not recommended), place it in 'password' element.
# Example: password: qwerty
# Password could be empty.
#
# If you want to specify SHA256, place it in 'password_sha256_hex' element.
# Example: password_sha256_hex: 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5
# Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
#
# If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
# Example: password_double_sha1_hex: e395796d6546b1b65db9d665cd43f0e858dd4303
#
# If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
# place its name in 'server' element inside 'ldap' element.
# Example: ldap:
# server: my_ldap_server
#
# If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
# place 'kerberos' element instead of 'password' (and similar) elements.
# The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
# You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
# whose initiator's realm matches it.
# Example: kerberos: ''
# Example: kerberos:
# realm: EXAMPLE.COM
#
# How to generate decent password:
# Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
# In first line will be password and in second - corresponding SHA256.
#
# How to generate double SHA1:
# Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
# In first line will be password and in second - corresponding double SHA1.
password: ''
# List of networks with open access.
#
# To open access from everywhere, specify:
# - ip: '::/0'
#
# To open access only from localhost, specify:
# - ip: '::1'
# - ip: 127.0.0.1
#
# Each element of list has one of the following forms:
# ip: IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
# 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
# host: Hostname. Example: server01.yandex.ru.
# To check access, DNS query is performed, and all received addresses compared to peer address.
# host_regexp: Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
# To check access, DNS PTR query is performed for peer address and then regexp is applied.
# Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
# Strongly recommended that regexp is ends with $ and take all expression in ''
# All results of DNS requests are cached till server restart.
networks:
ip: '::/0'
# Settings profile for user.
profile: default
# Quota for user.
quota: default
# User can create other users and grant rights to them.
# access_management: 1
# Quotas.
quotas:
# Name of quota.
default:
# Limits for time interval. You could specify many intervals with different limits.
interval:
# Length of interval.
duration: 3600
# No limits. Just calculate resource usage for time interval.
queries: 0
errors: 0
result_rows: 0
read_rows: 0
execution_time: 0

4
src/Common/tests/gtest_thread_pool_global_full.cpp
View File

@ -25,7 +25,7 @@ TEST(ThreadPool, GlobalFull1)
std::atomic<size_t> counter = 0; std::atomic<size_t> counter = 0;
static constexpr size_t num_jobs = capacity + 1; static constexpr size_t num_jobs = capacity + 1;
auto func = [&] { ++counter; while (counter != num_jobs) {} }; auto func = [&] { ++counter; while (counter != num_jobs) {} }; //-V776
ThreadPool pool(num_jobs); ThreadPool pool(num_jobs);
@ -63,7 +63,7 @@ TEST(ThreadPool, GlobalFull2)
global_pool.wait(); global_pool.wait();
std::atomic<size_t> counter = 0; std::atomic<size_t> counter = 0;
auto func = [&] { ++counter; while (counter != capacity + 1) {} }; auto func = [&] { ++counter; while (counter != capacity + 1) {} }; //-V776
ThreadPool pool(capacity, 0, capacity); ThreadPool pool(capacity, 0, capacity);
for (size_t i = 0; i < capacity; ++i) for (size_t i = 0; i < capacity; ++i)

1
src/Common/ya.make
View File

@ -33,6 +33,7 @@ SRCS(
Config/AbstractConfigurationComparison.cpp Config/AbstractConfigurationComparison.cpp
Config/ConfigProcessor.cpp Config/ConfigProcessor.cpp
Config/ConfigReloader.cpp Config/ConfigReloader.cpp
Config/YAMLParser.cpp
Config/configReadClient.cpp Config/configReadClient.cpp
CurrentMemoryTracker.cpp CurrentMemoryTracker.cpp
CurrentMetrics.cpp CurrentMetrics.cpp

64
src/Functions/FunctionsLogical.cpp
View File

@ -521,6 +521,70 @@ ColumnPtr FunctionAnyArityLogical<Impl, Name>::executeImpl(
return basicExecuteImpl<Impl>(std::move(args_in), input_rows_count); return basicExecuteImpl<Impl>(std::move(args_in), input_rows_count);
} }
template <typename Impl, typename Name>
ColumnPtr FunctionAnyArityLogical<Impl, Name>::getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const
{
/** Try to perform optimization for saturable functions (AndFunction, OrFunction) in case some arguments are
* constants.
* If function is not saturable (XorFunction) we cannot perform such optimization.
* If function is AndFunction and in arguments there is constant false, result is false.
* If function is OrFunction and in arguments there is constant true, result is true.
*/
if constexpr (!Impl::isSaturable())
return nullptr;
bool has_true_constant = false;
bool has_false_constant = false;
for (const auto & argument : arguments)
{
ColumnPtr column = argument.column;
if (!column || !isColumnConst(*column))
continue;
DataTypePtr non_nullable_type = removeNullable(argument.type);
TypeIndex data_type_index = non_nullable_type->getTypeId();
if (!isNativeNumber(data_type_index))
continue;
const ColumnConst * const_column = static_cast<const ColumnConst *>(column.get());
Field constant_field_value = const_column->getField();
if (constant_field_value.isNull())
continue;
auto field_type = constant_field_value.getType();
bool constant_value_bool = false;
if (field_type == Field::Types::Float64)
constant_value_bool = static_cast<bool>(constant_field_value.get<Float64>());
else if (field_type == Field::Types::Int64)
constant_value_bool = static_cast<bool>(constant_field_value.get<Int64>());
else if (field_type == Field::Types::UInt64)
constant_value_bool = static_cast<bool>(constant_field_value.get<UInt64>());
has_true_constant = has_true_constant || constant_value_bool;
has_false_constant = has_false_constant || !constant_value_bool;
}
ColumnPtr result_column;
if constexpr (std::is_same_v<Impl, AndImpl>)
{
if (has_false_constant)
result_type->createColumnConst(0, static_cast<UInt8>(false));
}
else if constexpr (std::is_same_v<Impl, OrImpl>)
{
if (has_true_constant)
result_type->createColumnConst(0, static_cast<UInt8>(true));
}
return result_column;
}
template <typename A, typename Op> template <typename A, typename Op>
struct UnaryOperationImpl struct UnaryOperationImpl

4
src/Functions/FunctionsLogical.h
View File

@ -155,7 +155,9 @@ public:
/// Get result types by argument types. If the function does not apply to these arguments, throw an exception. /// Get result types by argument types. If the function does not apply to these arguments, throw an exception.
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override;
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override; ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override;
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override;
#if USE_EMBEDDED_COMPILER #if USE_EMBEDDED_COMPILER
bool isCompilableImpl(const DataTypes &) const override { return useDefaultImplementationForNulls(); } bool isCompilableImpl(const DataTypes &) const override { return useDefaultImplementationForNulls(); }

13
src/Functions/IFunction.h
View File

@ -155,12 +155,13 @@ public:
*/ */
virtual bool isSuitableForConstantFolding() const { return true; } virtual bool isSuitableForConstantFolding() const { return true; }
/** Some functions like ignore(...) or toTypeName(...) always return constant result which doesn't depend on arguments. /** If function isSuitableForConstantFolding than, this method will be called during query analyzis
* In this case we can calculate result and assume that it's constant in stream header. * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can
* There is no need to implement function if it has zero arguments. * return they result based on some constant arguments.
* Must return ColumnConst with single row or nullptr. * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants,
* useDefaultImplementationForLowCardinality are not applied.
*/ */
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*columns*/) const { return nullptr; } virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
/** Function is called "injective" if it returns different result for different values of arguments. /** Function is called "injective" if it returns different result for different values of arguments.
* Example: hex, negate, tuple... * Example: hex, negate, tuple...
@ -377,7 +378,7 @@ public:
/// Properties from IFunctionBase (see IFunction.h) /// Properties from IFunctionBase (see IFunction.h)
virtual bool isSuitableForConstantFolding() const { return true; } virtual bool isSuitableForConstantFolding() const { return true; }
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; } virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*result_type*/) const { return nullptr; }
virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; } virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; }
virtual bool isDeterministic() const { return true; } virtual bool isDeterministic() const { return true; }
virtual bool isDeterministicInScopeOfQuery() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; }

5
src/Functions/IFunctionAdaptors.h
View File

@ -66,9 +66,10 @@ public:
} }
bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); } bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments_) const override
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments_, const DataTypePtr & result_type_) const override
{ {
return function->getResultIfAlwaysReturnsConstantAndHasArguments(arguments_); return function->getConstantResultForNonConstArguments(arguments_, result_type_);
} }
bool isStateful() const override { return function->isStateful(); } bool isStateful() const override { return function->isStateful(); }

2
src/Functions/defaultValueOfArgumentType.cpp
View File

@ -42,7 +42,7 @@ public:
return type.createColumnConst(input_rows_count, type.getDefault()); return type.createColumnConst(input_rows_count, type.getDefault());
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
{ {
const IDataType & type = *arguments[0].type; const IDataType & type = *arguments[0].type;
return type.createColumnConst(1, type.getDefault()); return type.createColumnConst(1, type.getDefault());

21
src/Functions/getSizeOfEnumType.cpp
View File

@ -51,17 +51,24 @@ public:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{ {
return getResultIfAlwaysReturnsConstantAndHasArguments(arguments)->cloneResized(input_rows_count); return getSizeOfEnumType(arguments[0].type, input_rows_count);
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
{ {
if (const auto * type8 = checkAndGetDataType<DataTypeEnum8>(arguments[0].type.get())) return getSizeOfEnumType(arguments[0].type, 1);
return DataTypeUInt8().createColumnConst(1, type8->getValues().size()); }
else if (const auto * type16 = checkAndGetDataType<DataTypeEnum16>(arguments[0].type.get()))
return DataTypeUInt16().createColumnConst(1, type16->getValues().size()); private:
ColumnPtr getSizeOfEnumType(const DataTypePtr & data_type, size_t input_rows_count) const
{
if (const auto * type8 = checkAndGetDataType<DataTypeEnum8>(data_type.get()))
return DataTypeUInt8().createColumnConst(input_rows_count, type8->getValues().size());
else if (const auto * type16 = checkAndGetDataType<DataTypeEnum16>(data_type.get()))
return DataTypeUInt16().createColumnConst(input_rows_count, type16->getValues().size());
else else
throw Exception("The argument for function " + getName() + " must be Enum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The argument for function {} must be Enum", getName());
} }
}; };

5
src/Functions/ignore.cpp
View File

@ -49,11 +49,6 @@ public:
{ {
return DataTypeUInt8().createColumnConst(input_rows_count, 0u); return DataTypeUInt8().createColumnConst(input_rows_count, 0u);
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
{
return DataTypeUInt8().createColumnConst(1, 0u);
}
}; };
} }

4
src/Functions/indexHint.cpp
View File

@ -55,10 +55,6 @@ public:
return DataTypeUInt8().createColumnConst(input_rows_count, 1u); return DataTypeUInt8().createColumnConst(input_rows_count, 1u);
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
{
return DataTypeUInt8().createColumnConst(1, 1u);
}
}; };

81
src/Functions/timezoneOf.cpp
View File

@ -23,66 +23,12 @@ namespace
/** timezoneOf(x) - get the name of the timezone of DateTime data type. /** timezoneOf(x) - get the name of the timezone of DateTime data type.
* Example: Europe/Moscow. * Example: Europe/Moscow.
*/ */
class ExecutableFunctionTimezoneOf : public IExecutableFunction class FunctionTimezoneOf : public IFunction
{ {
public: public:
static constexpr auto name = "timezoneOf"; static constexpr auto name = "timezoneOf";
String getName() const override { return name; } String getName() const override { return name; }
static FunctionPtr create(ContextPtr) { return std::make_unique<FunctionTimezoneOf>(); }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
/// Execute the function on the columns.
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
return DataTypeString().createColumnConst(input_rows_count,
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
}
};
class BaseFunctionTimezoneOf : public IFunctionBase
{
public:
BaseFunctionTimezoneOf(DataTypes argument_types_, DataTypePtr return_type_)
: argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {}
static constexpr auto name = "timezoneOf";
String getName() const override { return name; }
bool isDeterministic() const override { return true; }
bool isDeterministicInScopeOfQuery() const override { return true; }
const DataTypes & getArgumentTypes() const override { return argument_types; }
const DataTypePtr & getResultType() const override { return return_type; }
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override
{
return std::make_unique<ExecutableFunctionTimezoneOf>();
}
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override
{
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
return DataTypeString().createColumnConst(1,
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
}
private:
DataTypes argument_types;
DataTypePtr return_type;
};
class FunctionTimezoneOfBuilder : public IFunctionOverloadResolver
{
public:
static constexpr auto name = "timezoneOf";
String getName() const override { return name; }
static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique<FunctionTimezoneOfBuilder>(); }
size_t getNumberOfArguments() const override { return 1; } size_t getNumberOfArguments() const override { return 1; }
@ -96,21 +42,32 @@ public:
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad argument for function {}, should be DateTime or DateTime64", name); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad argument for function {}, should be DateTime or DateTime64", name);
} }
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override
{
return std::make_unique<BaseFunctionTimezoneOf>(DataTypes{arguments[0].type}, return_type);
}
bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
return DataTypeString().createColumnConst(input_rows_count,
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
}
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
{
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
return DataTypeString().createColumnConst(1,
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
}
}; };
} }
void registerFunctionTimezoneOf(FunctionFactory & factory) void registerFunctionTimezoneOf(FunctionFactory & factory)
{ {
factory.registerFunction<FunctionTimezoneOfBuilder>(); factory.registerFunction<FunctionTimezoneOf>();
factory.registerAlias("timeZoneOf", "timezoneOf"); factory.registerAlias("timeZoneOf", "timezoneOf");
} }

2
src/Functions/toColumnTypeName.cpp
View File

@ -41,7 +41,7 @@ public:
return DataTypeString().createColumnConst(input_rows_count, arguments[0].column->getName()); return DataTypeString().createColumnConst(input_rows_count, arguments[0].column->getName());
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
{ {
return DataTypeString().createColumnConst(1, arguments[0].type->createColumn()->getName()); return DataTypeString().createColumnConst(1, arguments[0].type->createColumn()->getName());
} }

84
src/Functions/toTypeName.cpp
View File

@ -12,85 +12,55 @@ namespace
/** toTypeName(x) - get the type name /** toTypeName(x) - get the type name
* Returns name of IDataType instance (name of data type). * Returns name of IDataType instance (name of data type).
*/ */
class ExecutableFunctionToTypeName : public IExecutableFunction class FunctionToTypeName : public IFunction
{ {
public: public:
static constexpr auto name = "toTypeName"; static constexpr auto name = "toTypeName";
String getName() const override { return name; }
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionToTypeName>();
}
String getName() const override
{
return name;
}
bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
/// Execute the function on the columns. size_t getNumberOfArguments() const override
{
return 1;
}
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
{
return std::make_shared<DataTypeString>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{ {
return DataTypeString().createColumnConst(input_rows_count, arguments[0].type->getName()); return DataTypeString().createColumnConst(input_rows_count, arguments[0].type->getName());
} }
};
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
class BaseFunctionToTypeName : public IFunctionBase
{
public:
BaseFunctionToTypeName(DataTypes argument_types_, DataTypePtr return_type_)
: argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {}
static constexpr auto name = "toTypeName";
String getName() const override { return name; }
bool isDeterministic() const override { return true; }
bool isDeterministicInScopeOfQuery() const override { return true; }
const DataTypes & getArgumentTypes() const override { return argument_types; }
const DataTypePtr & getResultType() const override { return return_type; }
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override
{ {
return std::make_unique<ExecutableFunctionToTypeName>(); return DataTypeString().createColumnConst(1, arguments[0].type->getName());
} }
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
{
return DataTypeString().createColumnConst(1, argument_types.at(0)->getName());
}
private:
DataTypes argument_types;
DataTypePtr return_type;
};
class FunctionToTypeNameBuilder : public IFunctionOverloadResolver
{
public:
static constexpr auto name = "toTypeName";
String getName() const override { return name; }
static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique<FunctionToTypeNameBuilder>(); }
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared<DataTypeString>(); }
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override
{
DataTypes types;
types.reserve(arguments.size());
for (const auto & elem : arguments)
types.emplace_back(elem.type);
return std::make_unique<BaseFunctionToTypeName>(types, return_type);
}
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; }
}; };
} }
void registerFunctionToTypeName(FunctionFactory & factory) void registerFunctionToTypeName(FunctionFactory & factory)
{ {
factory.registerFunction<FunctionToTypeNameBuilder>(); factory.registerFunction<FunctionToTypeName>();
} }
} }

47
src/Interpreters/ActionsDAG.cpp
View File

@ -205,34 +205,31 @@ const ActionsDAG::Node & ActionsDAG::addFunction(
node.function = node.function_base->prepare(arguments); node.function = node.function_base->prepare(arguments);
/// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function. /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function.
if (all_const && node.function_base->isSuitableForConstantFolding()) if (node.function_base->isSuitableForConstantFolding())
{ {
size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size(); ColumnPtr column;
auto col = node.function->execute(arguments, node.result_type, num_rows, true);
if (all_const)
{
size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size();
column = node.function->execute(arguments, node.result_type, num_rows, true);
}
else
{
column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type);
}
/// If the result is not a constant, just in case, we will consider the result as unknown. /// If the result is not a constant, just in case, we will consider the result as unknown.
if (isColumnConst(*col)) if (column && isColumnConst(*column))
{ {
/// All constant (literal) columns in block are added with size 1. /// All constant (literal) columns in block are added with size 1.
/// But if there was no columns in block before executing a function, the result has size 0. /// But if there was no columns in block before executing a function, the result has size 0.
/// Change the size to 1. /// Change the size to 1.
if (col->empty()) if (column->empty())
col = col->cloneResized(1); column = column->cloneResized(1);
node.column = std::move(col); node.column = std::move(column);
}
}
/// Some functions like ignore(), indexHint() or getTypeName() always return constant result even if arguments are not constant.
/// We can't do constant folding, but can specify in sample block that function result is constant to avoid
/// unnecessary materialization.
if (!node.column && node.function_base->isSuitableForConstantFolding())
{
if (auto col = node.function_base->getResultIfAlwaysReturnsConstantAndHasArguments(arguments))
{
node.column = std::move(col);
node.allow_constant_folding = false;
} }
} }
@ -408,13 +405,10 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs)
for (auto & node : nodes) for (auto & node : nodes)
{ {
/// We cannot remove function with side effects even if it returns constant (e.g. ignore(...)).
bool prevent_constant_folding = node.column && isColumnConst(*node.column) && !node.allow_constant_folding;
/// We cannot remove arrayJoin because it changes the number of rows. /// We cannot remove arrayJoin because it changes the number of rows.
bool is_array_join = node.type == ActionType::ARRAY_JOIN; bool is_array_join = node.type == ActionType::ARRAY_JOIN;
bool must_keep_node = is_array_join || prevent_constant_folding; if (is_array_join && visited_nodes.count(&node) == 0)
if (must_keep_node && visited_nodes.count(&node) == 0)
{ {
visited_nodes.insert(&node); visited_nodes.insert(&node);
stack.push(&node); stack.push(&node);
@ -429,7 +423,7 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs)
auto * node = stack.top(); auto * node = stack.top();
stack.pop(); stack.pop();
if (!node->children.empty() && node->column && isColumnConst(*node->column) && node->allow_constant_folding) if (!node->children.empty() && node->column && isColumnConst(*node->column))
{ {
/// Constant folding. /// Constant folding.
node->type = ActionsDAG::ActionType::COLUMN; node->type = ActionsDAG::ActionType::COLUMN;
@ -540,7 +534,7 @@ Block ActionsDAG::updateHeader(Block header) const
struct Frame struct Frame
{ {
const Node * node; const Node * node = nullptr;
size_t next_child = 0; size_t next_child = 0;
}; };
@ -587,8 +581,7 @@ Block ActionsDAG::updateHeader(Block header) const
} }
} }
auto & column = node_to_column[output]; if (node_to_column[output].column)
if (column.column)
result_columns.push_back(node_to_column[output]); result_columns.push_back(node_to_column[output]);
} }
} }

3
src/Interpreters/ActionsDAG.h
View File

@ -88,9 +88,6 @@ public:
/// For COLUMN node and propagated constants. /// For COLUMN node and propagated constants.
ColumnPtr column; ColumnPtr column;
/// Some functions like `ignore()` always return constant but can't be replaced by constant it.
/// We calculate such constants in order to avoid unnecessary materialization, but prohibit it's folding.
bool allow_constant_folding = true;
void toTree(JSONBuilder::JSONMap & map) const; void toTree(JSONBuilder::JSONMap & map) const;
}; };

2
src/Interpreters/ExpressionJIT.cpp
View File

@ -312,7 +312,7 @@ static FunctionBasePtr compile(
static bool isCompilableConstant(const ActionsDAG::Node & node) static bool isCompilableConstant(const ActionsDAG::Node & node)
{ {
return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding; return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type);
} }
static bool isCompilableFunction(const ActionsDAG::Node & node) static bool isCompilableFunction(const ActionsDAG::Node & node)

2
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -263,7 +263,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
} }
/// We use global context here, because storages lifetime is bigger than query context lifetime /// We use global context here, because storages lifetime is bigger than query context lifetime
database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach); database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach); //-V560
} }
catch (...) catch (...)
{ {

2
src/Parsers/New/ParserErrorListener.cpp
View File

@ -26,7 +26,7 @@ void ParserErrorListener::syntaxError(
auto * parser = dynamic_cast<ClickHouseParser *>(recognizer); auto * parser = dynamic_cast<ClickHouseParser *>(recognizer);
assert(parser); assert(parser);
LOG_ERROR(&Poco::Logger::get("ClickHouseParser"), LOG_ERROR(&Poco::Logger::get("ClickHouseParser"), //-V522
"Last element parsed so far:\n" "Last element parsed so far:\n"
"{}\n" "{}\n"
"Parser error: (pos {}) {}", parser->getRuleContext()->toStringTree(parser, true), token->getStartIndex(), message); "Parser error: (pos {}) {}", parser->getRuleContext()->toStringTree(parser, true), token->getStartIndex(), message);

8
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -578,6 +578,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
MergeTreeDataSelectSamplingData sampling = use_cache ? std::move(cache->sampling) : MergeTreeDataSelectSamplingData{}; MergeTreeDataSelectSamplingData sampling = use_cache ? std::move(cache->sampling) : MergeTreeDataSelectSamplingData{};
if (!use_cache) if (!use_cache)
{ {
assert(key_condition.has_value());
RelativeSize relative_sample_size = 0; RelativeSize relative_sample_size = 0;
RelativeSize relative_sample_offset = 0; RelativeSize relative_sample_offset = 0;
@ -606,7 +608,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
/// read) into the relative `SAMPLE 0.1` (how much data to read). /// read) into the relative `SAMPLE 0.1` (how much data to read).
size_t approx_total_rows = 0; size_t approx_total_rows = 0;
if (relative_sample_size > 1 || relative_sample_offset > 1) if (relative_sample_size > 1 || relative_sample_offset > 1)
approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); //-V1007
if (relative_sample_size > 1) if (relative_sample_size > 1)
{ {
@ -765,7 +767,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
if (has_lower_limit) if (has_lower_limit)
{ {
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) //-V1007
throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN);
ASTPtr args = std::make_shared<ASTExpressionList>(); ASTPtr args = std::make_shared<ASTExpressionList>();
@ -782,7 +784,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
if (has_upper_limit) if (has_upper_limit)
{ {
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) //-V1007
throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN);
ASTPtr args = std::make_shared<ASTExpressionList>(); ASTPtr args = std::make_shared<ASTExpressionList>();

2
src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
View File

@ -15,7 +15,7 @@ class KeyCondition;
struct MergeTreeDataSelectSamplingData struct MergeTreeDataSelectSamplingData
{ {
bool use_sampling; bool use_sampling = false;
std::shared_ptr<ASTFunction> filter_function; std::shared_ptr<ASTFunction> filter_function;
ActionsDAGPtr filter_expression; ActionsDAGPtr filter_expression;
}; };

2
src/Storages/MergeTree/MergeTreeReaderWide.cpp
View File

@ -173,7 +173,7 @@ void MergeTreeReaderWide::readData(
{ {
auto get_stream_getter = [&](bool stream_for_prefix) -> ISerialization::InputStreamGetter auto get_stream_getter = [&](bool stream_for_prefix) -> ISerialization::InputStreamGetter
{ {
return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * //-V1047
{ {
/// If substream have already been read. /// If substream have already been read.
if (cache.count(ISerialization::getSubcolumnNameForStream(substream_path))) if (cache.count(ISerialization::getSubcolumnNameForStream(substream_path)))

2
src/Storages/SelectQueryInfo.h
View File

@ -122,7 +122,7 @@ struct MergeTreeDataSelectCache;
// The projection selected to execute current query // The projection selected to execute current query
struct ProjectionCandidate struct ProjectionCandidate
{ {
const ProjectionDescription * desc; const ProjectionDescription * desc{};
PrewhereInfoPtr prewhere_info; PrewhereInfoPtr prewhere_info;
ActionsDAGPtr before_where; ActionsDAGPtr before_where;
String where_column_name; String where_column_name;

2
src/Storages/StorageLog.cpp
View File

@ -172,7 +172,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, ColumnPtr & colu
auto create_stream_getter = [&](bool stream_for_prefix) auto create_stream_getter = [&](bool stream_for_prefix)
{ {
return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * //-V1047
{ {
if (cache.count(ISerialization::getSubcolumnNameForStream(path))) if (cache.count(ISerialization::getSubcolumnNameForStream(path)))
return nullptr; return nullptr;

1
tests/clickhouse-test
View File

@ -38,6 +38,7 @@ MESSAGES_TO_RETRY = [
"Coordination::Exception: Operation timeout", "Coordination::Exception: Operation timeout",
"Operation timed out", "Operation timed out",
"ConnectionPoolWithFailover: Connection failed at try", "ConnectionPoolWithFailover: Connection failed at try",
"DB::Exception: New table appeared in database being dropped or detached. Try again"
] ]
class Terminated(KeyboardInterrupt): class Terminated(KeyboardInterrupt):

2
tests/performance/agg_functions_min_max_any.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.3">
<preconditions> <preconditions>
<table_exists>hits_100m_single</table_exists> <table_exists>hits_100m_single</table_exists>
</preconditions> </preconditions>

2
tests/performance/arithmetic.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test max_ignored_relative_change="0.7">
<settings> <settings>
<max_memory_usage>30000000000</max_memory_usage> <max_memory_usage>30000000000</max_memory_usage>
</settings> </settings>

2
tests/performance/array_join.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>

2
tests/performance/avg_weighted.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.4"> <test>
<settings> <settings>
<allow_experimental_bigint_types>1</allow_experimental_bigint_types> <allow_experimental_bigint_types>1</allow_experimental_bigint_types>
<max_threads>1</max_threads> <max_threads>1</max_threads>

2
tests/performance/bounding_ratio.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<query>SELECT boundingRatio(number, number) FROM numbers(100000000)</query> <query>SELECT boundingRatio(number, number) FROM numbers(100000000)</query>
<query>SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(100000000)</query> <query>SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(100000000)</query>
</test> </test>

3
tests/performance/codecs_float_insert.xml
View File

@ -1,5 +1,4 @@
<!-- FIXME this instability is abysmal, investigate the unstable queries --> <test>
<test max_ignored_relative_change="0.2">
<settings> <settings>
<allow_suspicious_codecs>1</allow_suspicious_codecs> <allow_suspicious_codecs>1</allow_suspicious_codecs>
</settings> </settings>

2
tests/performance/codecs_int_insert.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<settings> <settings>
<allow_suspicious_codecs>1</allow_suspicious_codecs> <allow_suspicious_codecs>1</allow_suspicious_codecs>
</settings> </settings>

2
tests/performance/collations.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>

2
tests/performance/date_time_64.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<preconditions> <preconditions>
<table_exists>hits_100m_single</table_exists> <table_exists>hits_100m_single</table_exists>
</preconditions> </preconditions>

2
tests/performance/date_time_long.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test>
<substitutions> <substitutions>
<substitution> <substitution>
<name>datetime_transform</name> <name>datetime_transform</name>

2
tests/performance/decimal_aggregates.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.5"> <test max_ignored_relative_change="0.2">
<settings> <settings>
<max_memory_usage>35G</max_memory_usage> <max_memory_usage>35G</max_memory_usage>
</settings> </settings>

2
tests/performance/decimal_casts.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.2">
<settings> <settings>
<max_memory_usage>15G</max_memory_usage> <max_memory_usage>15G</max_memory_usage>
</settings> </settings>

2
tests/performance/direct_dictionary.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.4">
<create_query> <create_query>
CREATE TABLE simple_key_direct_dictionary_source_table CREATE TABLE simple_key_direct_dictionary_source_table
( (

4
tests/performance/encodeXMLComponent.xml
View File

@ -1,7 +1,7 @@
<test> <test>
<preconditions> <preconditions>
<table_exists>test.hits</table_exists> <table_exists>hits_100m_single</table_exists>
</preconditions> </preconditions>
<query>SELECT count() FROM test.hits WHERE NOT ignore(encodeXMLComponent(URL))</query> <query>SELECT count() FROM hits_100m_single WHERE NOT ignore(encodeXMLComponent(URL))</query>
</test> </test>

2
tests/performance/flat_dictionary.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test>
<create_query> <create_query>
CREATE TABLE simple_key_flat_dictionary_source_table CREATE TABLE simple_key_flat_dictionary_source_table
( (

2
tests/performance/float_parsing.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<substitutions> <substitutions>
<substitution> <substitution>
<name>expr</name> <name>expr</name>

2
tests/performance/fuzz_bits.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test>

2
tests/performance/general_purpose_hashes.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.2">
<substitutions> <substitutions>
<substitution> <substitution>
<name>gp_hash_func</name> <name>gp_hash_func</name>

2
tests/performance/general_purpose_hashes_on_UUID.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.6">
<substitutions> <substitutions>
<substitution> <substitution>
<name>hash_func</name> <name>hash_func</name>

2
tests/performance/generate_table_function.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1000000000);</query> <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1000000000);</query>
<query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 1000000000);</query> <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 1000000000);</query>
<query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000);</query> <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000);</query>

2
tests/performance/hashed_dictionary.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.2">
<create_query> <create_query>
CREATE TABLE simple_key_hashed_dictionary_source_table CREATE TABLE simple_key_hashed_dictionary_source_table
( (

2
tests/performance/int_parsing.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<preconditions> <preconditions>
<table_exists>hits_100m_single</table_exists> <table_exists>hits_100m_single</table_exists>
<table_exists>hits_10m_single</table_exists> <table_exists>hits_10m_single</table_exists>

2
tests/performance/jit_small_requests.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<query> <query>
WITH WITH
bitXor(number, 0x4CF2D2BAAE6DA887) AS x0, bitXor(number, 0x4CF2D2BAAE6DA887) AS x0,

2
tests/performance/json_extract_rapidjson.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>

2
tests/performance/json_extract_simdjson.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.2">
<substitutions> <substitutions>
<substitution> <substitution>
<name>json</name> <name>json</name>

2
tests/performance/math.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.6">
<substitutions> <substitutions>
<substitution> <substitution>
<name>func_slow</name> <name>func_slow</name>

2
tests/performance/merge_tree_many_partitions_2.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<create_query>CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x</create_query> <create_query>CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x</create_query>
<fill_query>INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000)</fill_query> <fill_query>INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000)</fill_query>

2
tests/performance/number_formatting_formats.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test max_ignored_relative_change="0.3">
<substitutions> <substitutions>
<substitution> <substitution>
<name>format</name> <name>format</name>

2
tests/performance/optimized_select_final.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test>
<settings> <settings>
<do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final> <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
</settings> </settings>

2
tests/performance/parallel_mv.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<settings> <settings>
<parallel_view_processing>1</parallel_view_processing> <parallel_view_processing>1</parallel_view_processing>
</settings> </settings>

2
tests/performance/point_in_polygon.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.2">
<settings> <settings>
<!-- <!--
Not sure why it's needed. Maybe it has something to do with the Not sure why it's needed. Maybe it has something to do with the

2
tests/performance/polymorphic_parts_l.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<create_query> <create_query>
CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree() CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree()
PARTITION BY toYYYYMM(EventDate) PARTITION BY toYYYYMM(EventDate)

2
tests/performance/polymorphic_parts_m.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<create_query> <create_query>
CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree() CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree()
PARTITION BY toYYYYMM(EventDate) PARTITION BY toYYYYMM(EventDate)

2
tests/performance/polymorphic_parts_s.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<create_query> <create_query>
CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree() CREATE TABLE hits_wide AS hits_10m_single ENGINE = MergeTree()
PARTITION BY toYYYYMM(EventDate) PARTITION BY toYYYYMM(EventDate)

2
tests/performance/questdb_sum_int32.xml
View File

@ -1,4 +1,4 @@
<test> <test max_ignored_relative_change="0.2">
<settings> <settings>
<max_threads>4</max_threads> <max_threads>4</max_threads>
<max_memory_usage>20G</max_memory_usage> <max_memory_usage>20G</max_memory_usage>

2
tests/performance/random_string.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(10))</query> <query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(10))</query>
<query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(100))</query> <query>SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(100))</query>
<query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(1000))</query> <query>SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(1000))</query>

2
tests/performance/redundant_functions_in_order_by.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test>
<preconditions> <preconditions>
<table_exists>hits_100m_single</table_exists> <table_exists>hits_100m_single</table_exists>
</preconditions> </preconditions>

5
tests/performance/rewrite_sumIf.xml
View File

@ -1,4 +1,5 @@
<test> <test>
<query>SELECT sumIf(1, 0) FROM numbers(100000000)</query> <!-- Shouldn't have been a perf test, but an EXPLAIN one. -->
<query>SELECT sumIf(1, 1) FROM numbers(100000000)</query> <query>SELECT sumIf(1, 0) FROM numbers(1000000000)</query>
<query>SELECT sumIf(1, 1) FROM numbers(1000000000)</query>
</test> </test>

3
tests/performance/set_index.xml
View File

File diff suppressed because one or more lines are too long

2
tests/performance/sum_map.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.2">
<settings> <settings>
<max_threads>1</max_threads> <max_threads>1</max_threads>
</settings> </settings>

2
tests/performance/synthetic_hardware_benchmark.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.2">
<settings> <settings>
<max_memory_usage>30000000000</max_memory_usage> <max_memory_usage>30000000000</max_memory_usage>
</settings> </settings>

2
tests/performance/url_hits.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.2"> <test>
<preconditions> <preconditions>
<table_exists>hits_100m_single</table_exists> <table_exists>hits_100m_single</table_exists>
<table_exists>hits_10m_single</table_exists> <table_exists>hits_10m_single</table_exists>

2
tests/performance/visit_param_extract_raw.xml
View File

@ -1,4 +1,4 @@
<test max_ignored_relative_change="0.3"> <test max_ignored_relative_change="0.2">
<substitutions> <substitutions>
<substitution> <substitution>
<name>param</name> <name>param</name>