mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into tfs_extended_precision_data_types_testing
This commit is contained in:
MyroTk
commit
64573a9aa8
@ -29,7 +29,6 @@ message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}")
|
||||
|
||||
# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles.
|
||||
set (REQUIRED_LLVM_LIBRARIES
|
||||
LLVMOrcJIT
|
||||
LLVMExecutionEngine
|
||||
LLVMRuntimeDyld
|
||||
LLVMX86CodeGen
|
||||
|
||||
2
contrib/libunwind
vendored
2
contrib/libunwind
vendored
@ -1 +1 @@
|
||||
Subproject commit 8fe25d7dc70f2a4ea38c3e5a33fa9d4199b67a5a
|
||||
Subproject commit 1e4a2e5ce77be1af12e918a3c15dccf2bbac587d
|
||||
2
contrib/llvm
vendored
2
contrib/llvm
vendored
@ -1 +1 @@
|
||||
Subproject commit a7198805de67374eb3fb4c6b89797fa2d1cd7e50
|
||||
Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1
|
||||
@ -101,6 +101,8 @@ For very large clusters, you can use different ZooKeeper clusters for different
|
||||
|
||||
Replication is asynchronous and multi-master. `INSERT` queries (as well as `ALTER`) can be sent to any available server. Data is inserted on the server where the query is run, and then it is copied to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If part of the replicas are not available, the data is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. The number of threads performing background tasks for replicated tables can be set by [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) setting.
|
||||
|
||||
`ReplicatedMergeTree` engine uses a separate thread pool for replicated fetches. Size of the pool is limited by the [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) setting which can be tuned with a server restart.
|
||||
|
||||
By default, an INSERT query waits for confirmation of writing the data from only one replica. If the data was successfully written to only one replica and the server with this replica ceases to exist, the stored data will be lost. To enable getting confirmation of data writes from multiple replicas, use the `insert_quorum` option.
|
||||
|
||||
Each block of data is written atomically. The INSERT query is divided into blocks up to `max_insert_block_size = 1048576` rows. In other words, if the `INSERT` query has less than 1048576 rows, it is made atomically.
|
||||
@ -284,6 +286,7 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to
|
||||
**See Also**
|
||||
|
||||
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
|
||||
- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size)
|
||||
- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold)
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) <!--hide-->
|
||||
|
||||
@ -57,7 +57,7 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
|
||||
|
||||
## YAML examples {#example}
|
||||
|
||||
Here you can see default config written in YAML: [config-example.yaml](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config-example.yaml).
|
||||
Here you can see default config written in YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example).
|
||||
|
||||
There are some differences between YAML and XML formats in terms of ClickHouse configurations. Here are some tips for writing a configuration in YAML format.
|
||||
|
||||
|
||||
@ -2034,6 +2034,16 @@ Possible values:
|
||||
|
||||
Default value: 16.
|
||||
|
||||
## background_fetches_pool_size {#background_fetches_pool_size}
|
||||
|
||||
Sets the number of threads performing background fetches for [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. For production usage with frequent small insertions or slow ZooKeeper cluster is recomended to use default value.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 8.
|
||||
|
||||
## always_fetch_merged_part {#always_fetch_merged_part}
|
||||
|
||||
Prohibits data parts merging in [Replicated\*MergeTree](../../engines/table-engines/mergetree-family/replication.md)-engine tables.
|
||||
|
||||
@ -13,7 +13,7 @@ Returns an array of selected substrings. Empty substrings may be selected if the
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
splitByChar(<separator>, <s>)
|
||||
splitByChar(separator, s)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
@ -29,12 +29,12 @@ Returns an array of selected substrings. Empty substrings may be selected when:
|
||||
- There are multiple consecutive separators;
|
||||
- The original string `s` is empty.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md).
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT splitByChar(',', '1,2,3,abcde')
|
||||
SELECT splitByChar(',', '1,2,3,abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -50,7 +50,7 @@ Splits a string into substrings separated by a string. It uses a constant string
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
splitByString(<separator>, <s>)
|
||||
splitByString(separator, s)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
@ -62,7 +62,7 @@ splitByString(<separator>, <s>)
|
||||
|
||||
Returns an array of selected substrings. Empty substrings may be selected when:
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md).
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
- A non-empty separator occurs at the beginning or end of the string;
|
||||
- There are multiple consecutive non-empty separators;
|
||||
@ -71,7 +71,7 @@ Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-ref
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT splitByString(', ', '1, 2 3, 4,5, abcde')
|
||||
SELECT splitByString(', ', '1, 2 3, 4,5, abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -81,7 +81,7 @@ SELECT splitByString(', ', '1, 2 3, 4,5, abcde')
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT splitByString('', 'abcde')
|
||||
SELECT splitByString('', 'abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -92,12 +92,12 @@ SELECT splitByString('', 'abcde')
|
||||
|
||||
## splitByRegexp(regexp, s) {#splitbyregexpseparator-s}
|
||||
|
||||
Splits a string into substrings separated by a regular expression. It uses a regular expression string `regexp` as the separator. If the `regexp` is empty, it will split the string s into an array of single characters. If no match is found for this regex expression, the string `s` won't be split.
|
||||
Splits a string into substrings separated by a regular expression. It uses a regular expression string `regexp` as the separator. If the `regexp` is empty, it will split the string `s` into an array of single characters. If no match is found for this regular expression, the string `s` won't be split.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
splitByRegexp(<regexp>, <s>)
|
||||
splitByRegexp(regexp, s)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
@ -109,28 +109,36 @@ splitByRegexp(<regexp>, <s>)
|
||||
|
||||
Returns an array of selected substrings. Empty substrings may be selected when:
|
||||
|
||||
|
||||
- A non-empty regular expression match occurs at the beginning or end of the string;
|
||||
- There are multiple consecutive non-empty regular expression matches;
|
||||
- The original string `s` is empty while the regular expression is not empty.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md).
|
||||
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT splitByRegexp('\\d+', 'a12bc23de345f')
|
||||
SELECT splitByRegexp('\\d+', 'a12bc23de345f');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─splitByRegexp('\\d+', 'a12bc23de345f')─┐
|
||||
│ ['a','bc','de','f'] │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT splitByRegexp('', 'abcde')
|
||||
SELECT splitByRegexp('', 'abcde');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─splitByRegexp('', 'abcde')─┐
|
||||
│ ['a','b','c','d','e'] │
|
||||
@ -149,7 +157,7 @@ Selects substrings of consecutive bytes from the ranges a-z and A-Z.Returns an a
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT alphaTokens('abca1abc')
|
||||
SELECT alphaTokens('abca1abc');
|
||||
```
|
||||
|
||||
``` text
|
||||
|
||||
@ -65,6 +65,8 @@ ClickHouse хранит метаинформацию о репликах в [Apa
|
||||
|
||||
Репликация асинхронная, мульти-мастер. Запросы `INSERT` и `ALTER` можно направлять на любой доступный сервер. Данные вставятся на сервер, где выполнен запрос, а затем скопируются на остальные серверы. В связи с асинхронностью, только что вставленные данные появляются на остальных репликах с небольшой задержкой. Если часть реплик недоступна, данные на них запишутся тогда, когда они станут доступны. Если реплика доступна, то задержка составляет столько времени, сколько требуется для передачи блока сжатых данных по сети. Количество потоков для выполнения фоновых задач можно задать с помощью настройки [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size).
|
||||
|
||||
Движок `ReplicatedMergeTree` использует отдельный пул потоков для скачивания кусков данных. Размер пула ограничен настройкой [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size), которую можно указать при перезапуске сервера.
|
||||
|
||||
По умолчанию, запрос INSERT ждёт подтверждения записи только от одной реплики. Если данные были успешно записаны только на одну реплику, и сервер с этой репликой перестал существовать, то записанные данные будут потеряны. Вы можете включить подтверждение записи от нескольких реплик, используя настройку `insert_quorum`.
|
||||
|
||||
Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до `max_insert_block_size = 1048576` строк. То есть, если в запросе `INSERT` менее 1048576 строк, то он делается атомарно.
|
||||
@ -249,5 +251,6 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
|
||||
**Смотрите также**
|
||||
|
||||
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
|
||||
- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size)
|
||||
- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold)
|
||||
|
||||
|
||||
@ -6,9 +6,9 @@ toc_title: "Конфигурационные файлы"
|
||||
|
||||
# Конфигурационные файлы {#configuration_files}
|
||||
|
||||
Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`.
|
||||
Основной конфигурационный файл сервера - `config.xml` или `config.yaml`. Он расположен в директории `/etc/clickhouse-server/`.
|
||||
|
||||
Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf` из директории `config.d` рядом с конфигом.
|
||||
Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf`, а также `.yaml` (для файлов в формате YAML) из директории `config.d` рядом с конфигом.
|
||||
|
||||
У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`.
|
||||
|
||||
@ -25,7 +25,7 @@ toc_title: "Конфигурационные файлы"
|
||||
В элементе `users_config` файла `config.xml` можно указать относительный путь к конфигурационному файлу с настройками пользователей, профилей и квот. Значение `users_config` по умолчанию — `users.xml`. Если `users_config` не указан, то настройки пользователей, профилей и квот можно задать непосредственно в `config.xml`.
|
||||
|
||||
Настройки пользователя могут быть разделены в несколько отдельных файлов аналогичных `config.xml` и `config.d\`. Имя директории задаётся также как `users_config`.
|
||||
Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`.
|
||||
Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`/`.yaml`.
|
||||
Директория `users.d` используется по умолчанию, также как `users.xml` используется для `users_config`.
|
||||
Например, можно иметь по отдельному конфигурационному файлу для каждого пользователя:
|
||||
|
||||
@ -52,3 +52,66 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
|
||||
|
||||
Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера.
|
||||
|
||||
## Примеры записи конфигурации на YAML {#example}
|
||||
|
||||
Здесь можно рассмотреть пример реальной конфигурации записанной на YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example).
|
||||
|
||||
Между стандартами XML и YAML имеются различия, поэтому в этом разделе будут перечислены некоторые подсказки для написания конфигурации на YMAL.
|
||||
|
||||
Для записи обычной пары ключ-значение следует использовать Scalar:
|
||||
``` yaml
|
||||
key: value
|
||||
```
|
||||
|
||||
Для создания тега, содержащего подтеги следует использовать Map:
|
||||
``` yaml
|
||||
map_key:
|
||||
key1: val1
|
||||
key2: val2
|
||||
key3: val3
|
||||
```
|
||||
|
||||
Для создания списка значений или подтегов, расположенных по определенному ключу, следует использовать Sequence:
|
||||
``` yaml
|
||||
seq_key:
|
||||
- val1
|
||||
- val2
|
||||
- key1: val3
|
||||
- map:
|
||||
key2: val4
|
||||
key3: val5
|
||||
```
|
||||
|
||||
В случае, усли необходимо объявить тег, аналогичный XML-атрибуту, необходимо задать скаляр, имеющий ключ с префиксом @ и заключенный в кавычки:
|
||||
|
||||
``` yaml
|
||||
map:
|
||||
"@attr1": value1
|
||||
"@attr2": value2
|
||||
key: 123
|
||||
```
|
||||
|
||||
Из такой Map мы получим после конвертации:
|
||||
|
||||
``` xml
|
||||
<map attr1="value1" attr2="value2">
|
||||
<key>123</key>
|
||||
</map>
|
||||
```
|
||||
|
||||
Помимо Map, можно задавать атрибуты для Sequence:
|
||||
|
||||
``` yaml
|
||||
seq:
|
||||
- "@attr1": value1
|
||||
- "@attr2": value2
|
||||
- 123
|
||||
- abc
|
||||
```
|
||||
|
||||
Таким образом получая аналог следующей записи на XML:
|
||||
|
||||
``` xml
|
||||
<seq attr1="value1" attr2="value2">123</seq>
|
||||
<seq attr1="value1" attr2="value2">abc</seq>
|
||||
```
|
||||
|
||||
@ -2043,6 +2043,16 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1;
|
||||
|
||||
Значение по умолчанию: 16.
|
||||
|
||||
## background_fetches_pool_size {#background_fetches_pool_size}
|
||||
|
||||
Задает количество потоков для скачивания кусков данных для [реплицируемых](../../engines/table-engines/mergetree-family/replication.md) таблиц. Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе. Для использования в продакшене с частыми небольшими вставками или медленным кластером ZooKeeper рекомендуется использовать значение по умолчанию.
|
||||
|
||||
Допустимые значения:
|
||||
|
||||
- Положительное целое число.
|
||||
|
||||
Значение по умолчанию: 8.
|
||||
|
||||
## background_distributed_schedule_pool_size {#background_distributed_schedule_pool_size}
|
||||
|
||||
Задает количество потоков для выполнения фоновых задач. Работает для таблиц с движком [Distributed](../../engines/table-engines/special/distributed.md). Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе.
|
||||
|
||||
@ -14,7 +14,7 @@ separator должен быть константной строкой из ро
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
splitByChar(<separator>, <s>)
|
||||
splitByChar(separator, s)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
@ -30,12 +30,12 @@ splitByChar(<separator>, <s>)
|
||||
- Задано несколько последовательных разделителей;
|
||||
- Исходная строка `s` пуста.
|
||||
|
||||
Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md).
|
||||
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Пример**
|
||||
|
||||
``` sql
|
||||
SELECT splitByChar(',', '1,2,3,abcde')
|
||||
SELECT splitByChar(',', '1,2,3,abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -67,12 +67,12 @@ splitByString(separator, s)
|
||||
- Задано несколько последовательных разделителей;
|
||||
- Исходная строка `s` пуста.
|
||||
|
||||
Тип: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md).
|
||||
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Примеры**
|
||||
|
||||
``` sql
|
||||
SELECT splitByString(', ', '1, 2 3, 4,5, abcde')
|
||||
SELECT splitByString(', ', '1, 2 3, 4,5, abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -82,7 +82,7 @@ SELECT splitByString(', ', '1, 2 3, 4,5, abcde')
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT splitByString('', 'abcde')
|
||||
SELECT splitByString('', 'abcde');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -91,6 +91,60 @@ SELECT splitByString('', 'abcde')
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## splitByRegexp(regexp, s) {#splitbyregexpseparator-s}
|
||||
|
||||
Разбивает строку на подстроки, разделенные регулярным выражением. В качестве разделителя используется строка регулярного выражения `regexp`. Если `regexp` пустая, функция разделит строку `s` на массив одиночных символов. Если для регулярного выражения совпадения не найдено, строка `s` не будет разбита.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
splitByRegexp(regexp, s)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `regexp` — регулярное выражение. Константа. [String](../data-types/string.md) или [FixedString](../data-types/fixedstring.md).
|
||||
- `s` — разбиваемая строка. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемые значения**
|
||||
|
||||
Возвращает массив выбранных подстрок. Пустая подстрока может быть возвращена, если:
|
||||
|
||||
- Непустое совпадение с регулярным выражением происходит в начале или конце строки;
|
||||
- Имеется несколько последовательных совпадений c непустым регулярным выражением;
|
||||
- Исходная строка `s` пуста, а регулярное выражение не пустое.
|
||||
|
||||
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
|
||||
|
||||
**Примеры**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT splitByRegexp('\\d+', 'a12bc23de345f');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─splitByRegexp('\\d+', 'a12bc23de345f')─┐
|
||||
│ ['a','bc','de','f'] │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT splitByRegexp('', 'abcde');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─splitByRegexp('', 'abcde')─┐
|
||||
│ ['a','b','c','d','e'] │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## arrayStringConcat(arr\[, separator\]) {#arraystringconcatarr-separator}
|
||||
|
||||
@ -106,7 +160,7 @@ separator - необязательный параметр, константна
|
||||
**Пример:**
|
||||
|
||||
``` sql
|
||||
SELECT alphaTokens('abca1abc')
|
||||
SELECT alphaTokens('abca1abc');
|
||||
```
|
||||
|
||||
``` text
|
||||
@ -114,4 +168,3 @@ SELECT alphaTokens('abca1abc')
|
||||
│ ['abca','abc'] │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
@ -983,7 +983,7 @@ int mainEntryClickHouseStop(int argc, char ** argv)
|
||||
desc.add_options()
|
||||
("help,h", "produce help message")
|
||||
("pid-path", po::value<std::string>()->default_value("/var/run/clickhouse-server"), "directory for pid file")
|
||||
("force", po::value<bool>()->default_value(false), "Stop with KILL signal instead of TERM")
|
||||
("force", po::bool_switch(), "Stop with KILL signal instead of TERM")
|
||||
;
|
||||
|
||||
po::variables_map options;
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
# NOTE: User and query level settings are set up in "users.xml" file.
|
||||
# This is an example of a configuration file "config.xml" rewritten in YAML
|
||||
# You can read this documentation for detailed information about YAML configuration:
|
||||
# https://clickhouse.tech/docs/en/operations/configuration-files/
|
||||
|
||||
# NOTE: User and query level settings are set up in "users.yaml" file.
|
||||
# If you have accidentally specified user-level settings here, server won't start.
|
||||
# You can either move the settings to the right place inside "users.xml" file
|
||||
# or add skip_check_for_incorrect_settings: 1 here.
|
||||
107
programs/server/users.yaml.example
Normal file
107
programs/server/users.yaml.example
Normal file
@ -0,0 +1,107 @@
|
||||
# Profiles of settings.
|
||||
profiles:
|
||||
# Default settings.
|
||||
default:
|
||||
# Maximum memory usage for processing single query, in bytes.
|
||||
max_memory_usage: 10000000000
|
||||
|
||||
# How to choose between replicas during distributed query processing.
|
||||
# random - choose random replica from set of replicas with minimum number of errors
|
||||
# nearest_hostname - from set of replicas with minimum number of errors, choose replica
|
||||
# with minimum number of different symbols between replica's hostname and local hostname (Hamming distance).
|
||||
# in_order - first live replica is chosen in specified order.
|
||||
# first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
|
||||
load_balancing: random
|
||||
|
||||
# Profile that allows only read queries.
|
||||
readonly:
|
||||
readonly: 1
|
||||
|
||||
# Users and ACL.
|
||||
users:
|
||||
# If user name was not specified, 'default' user is used.
|
||||
default:
|
||||
# Password could be specified in plaintext or in SHA256 (in hex format).
|
||||
#
|
||||
# If you want to specify password in plaintext (not recommended), place it in 'password' element.
|
||||
# Example: password: qwerty
|
||||
# Password could be empty.
|
||||
#
|
||||
# If you want to specify SHA256, place it in 'password_sha256_hex' element.
|
||||
# Example: password_sha256_hex: 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5
|
||||
# Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
|
||||
#
|
||||
# If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
|
||||
# Example: password_double_sha1_hex: e395796d6546b1b65db9d665cd43f0e858dd4303
|
||||
#
|
||||
# If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
|
||||
# place its name in 'server' element inside 'ldap' element.
|
||||
# Example: ldap:
|
||||
# server: my_ldap_server
|
||||
#
|
||||
# If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
|
||||
# place 'kerberos' element instead of 'password' (and similar) elements.
|
||||
# The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
|
||||
# You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
|
||||
# whose initiator's realm matches it.
|
||||
# Example: kerberos: ''
|
||||
# Example: kerberos:
|
||||
# realm: EXAMPLE.COM
|
||||
#
|
||||
# How to generate decent password:
|
||||
# Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
|
||||
# In first line will be password and in second - corresponding SHA256.
|
||||
#
|
||||
# How to generate double SHA1:
|
||||
# Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
|
||||
# In first line will be password and in second - corresponding double SHA1.
|
||||
|
||||
password: ''
|
||||
|
||||
# List of networks with open access.
|
||||
#
|
||||
# To open access from everywhere, specify:
|
||||
# - ip: '::/0'
|
||||
#
|
||||
# To open access only from localhost, specify:
|
||||
# - ip: '::1'
|
||||
# - ip: 127.0.0.1
|
||||
#
|
||||
# Each element of list has one of the following forms:
|
||||
# ip: IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
|
||||
# 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||
# host: Hostname. Example: server01.yandex.ru.
|
||||
# To check access, DNS query is performed, and all received addresses compared to peer address.
|
||||
# host_regexp: Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
|
||||
# To check access, DNS PTR query is performed for peer address and then regexp is applied.
|
||||
# Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
|
||||
# Strongly recommended that regexp is ends with $ and take all expression in ''
|
||||
# All results of DNS requests are cached till server restart.
|
||||
|
||||
networks:
|
||||
ip: '::/0'
|
||||
|
||||
# Settings profile for user.
|
||||
profile: default
|
||||
|
||||
# Quota for user.
|
||||
quota: default
|
||||
|
||||
# User can create other users and grant rights to them.
|
||||
# access_management: 1
|
||||
|
||||
# Quotas.
|
||||
quotas:
|
||||
# Name of quota.
|
||||
default:
|
||||
# Limits for time interval. You could specify many intervals with different limits.
|
||||
interval:
|
||||
# Length of interval.
|
||||
duration: 3600
|
||||
|
||||
# No limits. Just calculate resource usage for time interval.
|
||||
queries: 0
|
||||
errors: 0
|
||||
result_rows: 0
|
||||
read_rows: 0
|
||||
execution_time: 0
|
||||
@ -25,7 +25,7 @@ TEST(ThreadPool, GlobalFull1)
|
||||
std::atomic<size_t> counter = 0;
|
||||
static constexpr size_t num_jobs = capacity + 1;
|
||||
|
||||
auto func = [&] { ++counter; while (counter != num_jobs) {} };
|
||||
auto func = [&] { ++counter; while (counter != num_jobs) {} }; //-V776
|
||||
|
||||
ThreadPool pool(num_jobs);
|
||||
|
||||
@ -63,7 +63,7 @@ TEST(ThreadPool, GlobalFull2)
|
||||
global_pool.wait();
|
||||
|
||||
std::atomic<size_t> counter = 0;
|
||||
auto func = [&] { ++counter; while (counter != capacity + 1) {} };
|
||||
auto func = [&] { ++counter; while (counter != capacity + 1) {} }; //-V776
|
||||
|
||||
ThreadPool pool(capacity, 0, capacity);
|
||||
for (size_t i = 0; i < capacity; ++i)
|
||||
|
||||
@ -54,6 +54,7 @@ class DiskS3::AwsS3KeyKeeper : public std::list<Aws::Vector<Aws::S3::Model::Obje
|
||||
{
|
||||
public:
|
||||
void addKey(const String & key);
|
||||
static String getChunkKeys(const Aws::Vector<Aws::S3::Model::ObjectIdentifier> & chunk);
|
||||
|
||||
private:
|
||||
/// limit for one DeleteObject request
|
||||
@ -74,6 +75,19 @@ void DiskS3::AwsS3KeyKeeper::addKey(const String & key)
|
||||
back().push_back(obj);
|
||||
}
|
||||
|
||||
String DiskS3::AwsS3KeyKeeper::getChunkKeys(const Aws::Vector<Aws::S3::Model::ObjectIdentifier> & chunk)
|
||||
{
|
||||
String res;
|
||||
for (const auto & obj : chunk)
|
||||
{
|
||||
const auto & key = obj.GetKey();
|
||||
if (!res.empty())
|
||||
res.append(", ");
|
||||
res.append(key.c_str(), key.size());
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
String getRandomName()
|
||||
{
|
||||
std::uniform_int_distribution<int> distribution('a', 'z');
|
||||
@ -794,6 +808,8 @@ void DiskS3::removeAws(const AwsS3KeyKeeper & keys)
|
||||
|
||||
for (const auto & chunk : keys)
|
||||
{
|
||||
LOG_DEBUG(log, "Remove AWS keys {}", AwsS3KeyKeeper::getChunkKeys(chunk));
|
||||
|
||||
Aws::S3::Model::Delete delkeys;
|
||||
delkeys.SetObjects(chunk);
|
||||
|
||||
|
||||
@ -521,6 +521,70 @@ ColumnPtr FunctionAnyArityLogical<Impl, Name>::executeImpl(
|
||||
return basicExecuteImpl<Impl>(std::move(args_in), input_rows_count);
|
||||
}
|
||||
|
||||
template <typename Impl, typename Name>
|
||||
ColumnPtr FunctionAnyArityLogical<Impl, Name>::getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const
|
||||
{
|
||||
/** Try to perform optimization for saturable functions (AndFunction, OrFunction) in case some arguments are
|
||||
* constants.
|
||||
* If function is not saturable (XorFunction) we cannot perform such optimization.
|
||||
* If function is AndFunction and in arguments there is constant false, result is false.
|
||||
* If function is OrFunction and in arguments there is constant true, result is true.
|
||||
*/
|
||||
if constexpr (!Impl::isSaturable())
|
||||
return nullptr;
|
||||
|
||||
bool has_true_constant = false;
|
||||
bool has_false_constant = false;
|
||||
|
||||
for (const auto & argument : arguments)
|
||||
{
|
||||
ColumnPtr column = argument.column;
|
||||
|
||||
if (!column || !isColumnConst(*column))
|
||||
continue;
|
||||
|
||||
DataTypePtr non_nullable_type = removeNullable(argument.type);
|
||||
TypeIndex data_type_index = non_nullable_type->getTypeId();
|
||||
|
||||
if (!isNativeNumber(data_type_index))
|
||||
continue;
|
||||
|
||||
const ColumnConst * const_column = static_cast<const ColumnConst *>(column.get());
|
||||
|
||||
Field constant_field_value = const_column->getField();
|
||||
if (constant_field_value.isNull())
|
||||
continue;
|
||||
|
||||
auto field_type = constant_field_value.getType();
|
||||
|
||||
bool constant_value_bool = false;
|
||||
|
||||
if (field_type == Field::Types::Float64)
|
||||
constant_value_bool = static_cast<bool>(constant_field_value.get<Float64>());
|
||||
else if (field_type == Field::Types::Int64)
|
||||
constant_value_bool = static_cast<bool>(constant_field_value.get<Int64>());
|
||||
else if (field_type == Field::Types::UInt64)
|
||||
constant_value_bool = static_cast<bool>(constant_field_value.get<UInt64>());
|
||||
|
||||
has_true_constant = has_true_constant || constant_value_bool;
|
||||
has_false_constant = has_false_constant || !constant_value_bool;
|
||||
}
|
||||
|
||||
ColumnPtr result_column;
|
||||
|
||||
if constexpr (std::is_same_v<Impl, AndImpl>)
|
||||
{
|
||||
if (has_false_constant)
|
||||
result_type->createColumnConst(0, static_cast<UInt8>(false));
|
||||
}
|
||||
else if constexpr (std::is_same_v<Impl, OrImpl>)
|
||||
{
|
||||
if (has_true_constant)
|
||||
result_type->createColumnConst(0, static_cast<UInt8>(true));
|
||||
}
|
||||
|
||||
return result_column;
|
||||
}
|
||||
|
||||
template <typename A, typename Op>
|
||||
struct UnaryOperationImpl
|
||||
|
||||
@ -155,7 +155,9 @@ public:
|
||||
/// Get result types by argument types. If the function does not apply to these arguments, throw an exception.
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override;
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override;
|
||||
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override;
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
bool isCompilableImpl(const DataTypes &) const override { return useDefaultImplementationForNulls(); }
|
||||
|
||||
@ -155,12 +155,13 @@ public:
|
||||
*/
|
||||
virtual bool isSuitableForConstantFolding() const { return true; }
|
||||
|
||||
/** Some functions like ignore(...) or toTypeName(...) always return constant result which doesn't depend on arguments.
|
||||
* In this case we can calculate result and assume that it's constant in stream header.
|
||||
* There is no need to implement function if it has zero arguments.
|
||||
* Must return ColumnConst with single row or nullptr.
|
||||
/** If function isSuitableForConstantFolding then, this method will be called during query analyzis
|
||||
* if some arguments are constants. For example logical functions (AndFunction, OrFunction) can
|
||||
* return they result based on some constant arguments.
|
||||
* Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants,
|
||||
* useDefaultImplementationForLowCardinality are not applied.
|
||||
*/
|
||||
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*columns*/) const { return nullptr; }
|
||||
virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
|
||||
|
||||
/** Function is called "injective" if it returns different result for different values of arguments.
|
||||
* Example: hex, negate, tuple...
|
||||
@ -300,7 +301,7 @@ protected:
|
||||
return getReturnTypeImpl(data_types);
|
||||
}
|
||||
|
||||
/** If useDefaultImplementationForNulls() is true, than change arguments for getReturnType() and build():
|
||||
/** If useDefaultImplementationForNulls() is true, then change arguments for getReturnType() and build():
|
||||
* if some of arguments are Nullable(Nothing) then don't call getReturnType(), call build() with return_type = Nullable(Nothing),
|
||||
* if some of arguments are Nullable, then:
|
||||
* - Nullable types are substituted with nested types for getReturnType() function
|
||||
@ -310,7 +311,7 @@ protected:
|
||||
*/
|
||||
virtual bool useDefaultImplementationForNulls() const { return true; }
|
||||
|
||||
/** If useDefaultImplementationForNulls() is true, than change arguments for getReturnType() and build().
|
||||
/** If useDefaultImplementationForNulls() is true, then change arguments for getReturnType() and build().
|
||||
* If function arguments has low cardinality types, convert them to ordinary types.
|
||||
* getReturnType returns ColumnLowCardinality if at least one argument type is ColumnLowCardinality.
|
||||
*/
|
||||
@ -377,7 +378,7 @@ public:
|
||||
|
||||
/// Properties from IFunctionBase (see IFunction.h)
|
||||
virtual bool isSuitableForConstantFolding() const { return true; }
|
||||
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; }
|
||||
virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*result_type*/) const { return nullptr; }
|
||||
virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; }
|
||||
virtual bool isDeterministic() const { return true; }
|
||||
virtual bool isDeterministicInScopeOfQuery() const { return true; }
|
||||
|
||||
@ -66,9 +66,10 @@ public:
|
||||
}
|
||||
|
||||
bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); }
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments_) const override
|
||||
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments_, const DataTypePtr & result_type_) const override
|
||||
{
|
||||
return function->getResultIfAlwaysReturnsConstantAndHasArguments(arguments_);
|
||||
return function->getConstantResultForNonConstArguments(arguments_, result_type_);
|
||||
}
|
||||
|
||||
bool isStateful() const override { return function->isStateful(); }
|
||||
|
||||
@ -42,7 +42,7 @@ public:
|
||||
return type.createColumnConst(input_rows_count, type.getDefault());
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
|
||||
{
|
||||
const IDataType & type = *arguments[0].type;
|
||||
return type.createColumnConst(1, type.getDefault());
|
||||
|
||||
@ -51,17 +51,24 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
return getResultIfAlwaysReturnsConstantAndHasArguments(arguments)->cloneResized(input_rows_count);
|
||||
return getSizeOfEnumType(arguments[0].type, input_rows_count);
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
|
||||
{
|
||||
if (const auto * type8 = checkAndGetDataType<DataTypeEnum8>(arguments[0].type.get()))
|
||||
return DataTypeUInt8().createColumnConst(1, type8->getValues().size());
|
||||
else if (const auto * type16 = checkAndGetDataType<DataTypeEnum16>(arguments[0].type.get()))
|
||||
return DataTypeUInt16().createColumnConst(1, type16->getValues().size());
|
||||
return getSizeOfEnumType(arguments[0].type, 1);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
ColumnPtr getSizeOfEnumType(const DataTypePtr & data_type, size_t input_rows_count) const
|
||||
{
|
||||
if (const auto * type8 = checkAndGetDataType<DataTypeEnum8>(data_type.get()))
|
||||
return DataTypeUInt8().createColumnConst(input_rows_count, type8->getValues().size());
|
||||
else if (const auto * type16 = checkAndGetDataType<DataTypeEnum16>(data_type.get()))
|
||||
return DataTypeUInt16().createColumnConst(input_rows_count, type16->getValues().size());
|
||||
else
|
||||
throw Exception("The argument for function " + getName() + " must be Enum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The argument for function {} must be Enum", getName());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -49,11 +49,6 @@ public:
|
||||
{
|
||||
return DataTypeUInt8().createColumnConst(input_rows_count, 0u);
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return DataTypeUInt8().createColumnConst(1, 0u);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -55,10 +55,6 @@ public:
|
||||
return DataTypeUInt8().createColumnConst(input_rows_count, 1u);
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return DataTypeUInt8().createColumnConst(1, 1u);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -23,66 +23,12 @@ namespace
|
||||
/** timezoneOf(x) - get the name of the timezone of DateTime data type.
|
||||
* Example: Europe/Moscow.
|
||||
*/
|
||||
class ExecutableFunctionTimezoneOf : public IExecutableFunction
|
||||
class FunctionTimezoneOf : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "timezoneOf";
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
|
||||
/// Execute the function on the columns.
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
|
||||
|
||||
return DataTypeString().createColumnConst(input_rows_count,
|
||||
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class BaseFunctionTimezoneOf : public IFunctionBase
|
||||
{
|
||||
public:
|
||||
BaseFunctionTimezoneOf(DataTypes argument_types_, DataTypePtr return_type_)
|
||||
: argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {}
|
||||
|
||||
static constexpr auto name = "timezoneOf";
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isDeterministic() const override { return true; }
|
||||
bool isDeterministicInScopeOfQuery() const override { return true; }
|
||||
|
||||
const DataTypes & getArgumentTypes() const override { return argument_types; }
|
||||
const DataTypePtr & getResultType() const override { return return_type; }
|
||||
|
||||
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return std::make_unique<ExecutableFunctionTimezoneOf>();
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
|
||||
|
||||
return DataTypeString().createColumnConst(1,
|
||||
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
|
||||
}
|
||||
|
||||
private:
|
||||
DataTypes argument_types;
|
||||
DataTypePtr return_type;
|
||||
};
|
||||
|
||||
|
||||
class FunctionTimezoneOfBuilder : public IFunctionOverloadResolver
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "timezoneOf";
|
||||
String getName() const override { return name; }
|
||||
static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique<FunctionTimezoneOfBuilder>(); }
|
||||
static FunctionPtr create(ContextPtr) { return std::make_unique<FunctionTimezoneOf>(); }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
@ -96,21 +42,32 @@ public:
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad argument for function {}, should be DateTime or DateTime64", name);
|
||||
}
|
||||
|
||||
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override
|
||||
{
|
||||
return std::make_unique<BaseFunctionTimezoneOf>(DataTypes{arguments[0].type}, return_type);
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; }
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
|
||||
|
||||
return DataTypeString().createColumnConst(input_rows_count,
|
||||
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
|
||||
}
|
||||
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
|
||||
{
|
||||
DataTypePtr type_no_nullable = removeNullable(arguments[0].type);
|
||||
|
||||
return DataTypeString().createColumnConst(1,
|
||||
dynamic_cast<const TimezoneMixin &>(*type_no_nullable).getTimeZone().getTimeZone());
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionTimezoneOf(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionTimezoneOfBuilder>();
|
||||
factory.registerFunction<FunctionTimezoneOf>();
|
||||
factory.registerAlias("timeZoneOf", "timezoneOf");
|
||||
}
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ public:
|
||||
return DataTypeString().createColumnConst(input_rows_count, arguments[0].column->getName());
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
|
||||
{
|
||||
return DataTypeString().createColumnConst(1, arguments[0].type->createColumn()->getName());
|
||||
}
|
||||
|
||||
@ -12,85 +12,55 @@ namespace
|
||||
/** toTypeName(x) - get the type name
|
||||
* Returns name of IDataType instance (name of data type).
|
||||
*/
|
||||
class ExecutableFunctionToTypeName : public IExecutableFunction
|
||||
class FunctionToTypeName : public IFunction
|
||||
{
|
||||
public:
|
||||
|
||||
static constexpr auto name = "toTypeName";
|
||||
String getName() const override { return name; }
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionToTypeName>();
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
|
||||
/// Execute the function on the columns.
|
||||
size_t getNumberOfArguments() const override
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
|
||||
{
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
return DataTypeString().createColumnConst(input_rows_count, arguments[0].type->getName());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class BaseFunctionToTypeName : public IFunctionBase
|
||||
ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override
|
||||
{
|
||||
public:
|
||||
BaseFunctionToTypeName(DataTypes argument_types_, DataTypePtr return_type_)
|
||||
: argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {}
|
||||
|
||||
static constexpr auto name = "toTypeName";
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isDeterministic() const override { return true; }
|
||||
bool isDeterministicInScopeOfQuery() const override { return true; }
|
||||
|
||||
const DataTypes & getArgumentTypes() const override { return argument_types; }
|
||||
const DataTypePtr & getResultType() const override { return return_type; }
|
||||
|
||||
ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return std::make_unique<ExecutableFunctionToTypeName>();
|
||||
return DataTypeString().createColumnConst(1, arguments[0].type->getName());
|
||||
}
|
||||
|
||||
ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override
|
||||
{
|
||||
return DataTypeString().createColumnConst(1, argument_types.at(0)->getName());
|
||||
}
|
||||
|
||||
private:
|
||||
DataTypes argument_types;
|
||||
DataTypePtr return_type;
|
||||
};
|
||||
|
||||
|
||||
class FunctionToTypeNameBuilder : public IFunctionOverloadResolver
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "toTypeName";
|
||||
String getName() const override { return name; }
|
||||
static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique<FunctionToTypeNameBuilder>(); }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared<DataTypeString>(); }
|
||||
|
||||
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override
|
||||
{
|
||||
DataTypes types;
|
||||
types.reserve(arguments.size());
|
||||
for (const auto & elem : arguments)
|
||||
types.emplace_back(elem.type);
|
||||
|
||||
return std::make_unique<BaseFunctionToTypeName>(types, return_type);
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; }
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionToTypeName(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionToTypeNameBuilder>();
|
||||
factory.registerFunction<FunctionToTypeName>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -205,34 +205,31 @@ const ActionsDAG::Node & ActionsDAG::addFunction(
|
||||
node.function = node.function_base->prepare(arguments);
|
||||
|
||||
/// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function.
|
||||
if (all_const && node.function_base->isSuitableForConstantFolding())
|
||||
if (node.function_base->isSuitableForConstantFolding())
|
||||
{
|
||||
ColumnPtr column;
|
||||
|
||||
if (all_const)
|
||||
{
|
||||
size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size();
|
||||
auto col = node.function->execute(arguments, node.result_type, num_rows, true);
|
||||
column = node.function->execute(arguments, node.result_type, num_rows, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type);
|
||||
}
|
||||
|
||||
/// If the result is not a constant, just in case, we will consider the result as unknown.
|
||||
if (isColumnConst(*col))
|
||||
if (column && isColumnConst(*column))
|
||||
{
|
||||
/// All constant (literal) columns in block are added with size 1.
|
||||
/// But if there was no columns in block before executing a function, the result has size 0.
|
||||
/// Change the size to 1.
|
||||
|
||||
if (col->empty())
|
||||
col = col->cloneResized(1);
|
||||
if (column->empty())
|
||||
column = column->cloneResized(1);
|
||||
|
||||
node.column = std::move(col);
|
||||
}
|
||||
}
|
||||
|
||||
/// Some functions like ignore(), indexHint() or getTypeName() always return constant result even if arguments are not constant.
|
||||
/// We can't do constant folding, but can specify in sample block that function result is constant to avoid
|
||||
/// unnecessary materialization.
|
||||
if (!node.column && node.function_base->isSuitableForConstantFolding())
|
||||
{
|
||||
if (auto col = node.function_base->getResultIfAlwaysReturnsConstantAndHasArguments(arguments))
|
||||
{
|
||||
node.column = std::move(col);
|
||||
node.allow_constant_folding = false;
|
||||
node.column = std::move(column);
|
||||
}
|
||||
}
|
||||
|
||||
@ -408,13 +405,10 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs)
|
||||
|
||||
for (auto & node : nodes)
|
||||
{
|
||||
/// We cannot remove function with side effects even if it returns constant (e.g. ignore(...)).
|
||||
bool prevent_constant_folding = node.column && isColumnConst(*node.column) && !node.allow_constant_folding;
|
||||
/// We cannot remove arrayJoin because it changes the number of rows.
|
||||
bool is_array_join = node.type == ActionType::ARRAY_JOIN;
|
||||
|
||||
bool must_keep_node = is_array_join || prevent_constant_folding;
|
||||
if (must_keep_node && visited_nodes.count(&node) == 0)
|
||||
if (is_array_join && visited_nodes.count(&node) == 0)
|
||||
{
|
||||
visited_nodes.insert(&node);
|
||||
stack.push(&node);
|
||||
@ -429,7 +423,7 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs)
|
||||
auto * node = stack.top();
|
||||
stack.pop();
|
||||
|
||||
if (!node->children.empty() && node->column && isColumnConst(*node->column) && node->allow_constant_folding)
|
||||
if (!node->children.empty() && node->column && isColumnConst(*node->column))
|
||||
{
|
||||
/// Constant folding.
|
||||
node->type = ActionsDAG::ActionType::COLUMN;
|
||||
@ -540,7 +534,7 @@ Block ActionsDAG::updateHeader(Block header) const
|
||||
|
||||
struct Frame
|
||||
{
|
||||
const Node * node;
|
||||
const Node * node = nullptr;
|
||||
size_t next_child = 0;
|
||||
};
|
||||
|
||||
@ -587,8 +581,7 @@ Block ActionsDAG::updateHeader(Block header) const
|
||||
}
|
||||
}
|
||||
|
||||
auto & column = node_to_column[output];
|
||||
if (column.column)
|
||||
if (node_to_column[output].column)
|
||||
result_columns.push_back(node_to_column[output]);
|
||||
}
|
||||
}
|
||||
@ -1521,7 +1514,7 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet &
|
||||
|
||||
auto res = split(split_nodes);
|
||||
/// Do not remove array joined columns if they are not used.
|
||||
res.first->project_input = false;
|
||||
/// res.first->project_input = false;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -1534,7 +1527,9 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co
|
||||
column_name, dumpDAG());
|
||||
|
||||
std::unordered_set<const Node *> split_nodes = {node};
|
||||
return split(split_nodes);
|
||||
auto res = split(split_nodes);
|
||||
res.second->project_input = project_input;
|
||||
return res;
|
||||
}
|
||||
|
||||
namespace
|
||||
|
||||
@ -88,9 +88,6 @@ public:
|
||||
|
||||
/// For COLUMN node and propagated constants.
|
||||
ColumnPtr column;
|
||||
/// Some functions like `ignore()` always return constant but can't be replaced by constant it.
|
||||
/// We calculate such constants in order to avoid unnecessary materialization, but prohibit it's folding.
|
||||
bool allow_constant_folding = true;
|
||||
|
||||
void toTree(JSONBuilder::JSONMap & map) const;
|
||||
};
|
||||
|
||||
@ -312,7 +312,7 @@ static FunctionBasePtr compile(
|
||||
|
||||
static bool isCompilableConstant(const ActionsDAG::Node & node)
|
||||
{
|
||||
return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding;
|
||||
return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type);
|
||||
}
|
||||
|
||||
static bool isCompilableFunction(const ActionsDAG::Node & node)
|
||||
|
||||
@ -263,7 +263,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
}
|
||||
|
||||
/// We use global context here, because storages lifetime is bigger than query context lifetime
|
||||
database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach);
|
||||
database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach); //-V560
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -26,7 +26,7 @@ void ParserErrorListener::syntaxError(
|
||||
auto * parser = dynamic_cast<ClickHouseParser *>(recognizer);
|
||||
assert(parser);
|
||||
|
||||
LOG_ERROR(&Poco::Logger::get("ClickHouseParser"),
|
||||
LOG_ERROR(&Poco::Logger::get("ClickHouseParser"), //-V522
|
||||
"Last element parsed so far:\n"
|
||||
"{}\n"
|
||||
"Parser error: (pos {}) {}", parser->getRuleContext()->toStringTree(parser, true), token->getStartIndex(), message);
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#include <Processors/QueryPlan/UnionStep.h>
|
||||
#include <Processors/QueryPipeline.h>
|
||||
#include <Processors/Sources/NullSource.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Interpreters/ExpressionActions.h>
|
||||
#include <common/defines.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -47,6 +49,28 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines, const Build
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
for (auto & cur_pipeline : pipelines)
|
||||
{
|
||||
#if !defined(NDEBUG)
|
||||
assertCompatibleHeader(cur_pipeline->getHeader(), getOutputStream().header, "UnionStep");
|
||||
#endif
|
||||
/// Headers for union must be equal.
|
||||
/// But, just in case, convert it to the same header if not.
|
||||
if (!isCompatibleHeader(cur_pipeline->getHeader(), getOutputStream().header))
|
||||
{
|
||||
auto converting_dag = ActionsDAG::makeConvertingActions(
|
||||
cur_pipeline->getHeader().getColumnsWithTypeAndName(),
|
||||
getOutputStream().header.getColumnsWithTypeAndName(),
|
||||
ActionsDAG::MatchColumnsMode::Name);
|
||||
|
||||
auto converting_actions = std::make_shared<ExpressionActions>(std::move(converting_dag));
|
||||
cur_pipeline->addSimpleTransform([&](const Block & cur_header)
|
||||
{
|
||||
return std::make_shared<ExpressionTransform>(cur_header, converting_actions);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
*pipeline = QueryPipeline::unitePipelines(std::move(pipelines), max_threads);
|
||||
|
||||
processors = collector.detachProcessors();
|
||||
|
||||
@ -578,6 +578,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
|
||||
MergeTreeDataSelectSamplingData sampling = use_cache ? std::move(cache->sampling) : MergeTreeDataSelectSamplingData{};
|
||||
if (!use_cache)
|
||||
{
|
||||
assert(key_condition.has_value());
|
||||
|
||||
RelativeSize relative_sample_size = 0;
|
||||
RelativeSize relative_sample_offset = 0;
|
||||
|
||||
@ -606,7 +608,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
|
||||
/// read) into the relative `SAMPLE 0.1` (how much data to read).
|
||||
size_t approx_total_rows = 0;
|
||||
if (relative_sample_size > 1 || relative_sample_offset > 1)
|
||||
approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings);
|
||||
approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); //-V1007
|
||||
|
||||
if (relative_sample_size > 1)
|
||||
{
|
||||
@ -765,7 +767,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
|
||||
|
||||
if (has_lower_limit)
|
||||
{
|
||||
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true)))
|
||||
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) //-V1007
|
||||
throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
ASTPtr args = std::make_shared<ASTExpressionList>();
|
||||
@ -782,7 +784,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
|
||||
|
||||
if (has_upper_limit)
|
||||
{
|
||||
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false)))
|
||||
if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) //-V1007
|
||||
throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
ASTPtr args = std::make_shared<ASTExpressionList>();
|
||||
|
||||
@ -15,7 +15,7 @@ class KeyCondition;
|
||||
|
||||
struct MergeTreeDataSelectSamplingData
|
||||
{
|
||||
bool use_sampling;
|
||||
bool use_sampling = false;
|
||||
std::shared_ptr<ASTFunction> filter_function;
|
||||
ActionsDAGPtr filter_expression;
|
||||
};
|
||||
|
||||
@ -173,7 +173,7 @@ void MergeTreeReaderWide::readData(
|
||||
{
|
||||
auto get_stream_getter = [&](bool stream_for_prefix) -> ISerialization::InputStreamGetter
|
||||
{
|
||||
return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer *
|
||||
return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * //-V1047
|
||||
{
|
||||
/// If substream have already been read.
|
||||
if (cache.count(ISerialization::getSubcolumnNameForStream(substream_path)))
|
||||
|
||||
@ -122,7 +122,7 @@ struct MergeTreeDataSelectCache;
|
||||
// The projection selected to execute current query
|
||||
struct ProjectionCandidate
|
||||
{
|
||||
const ProjectionDescription * desc;
|
||||
const ProjectionDescription * desc{};
|
||||
PrewhereInfoPtr prewhere_info;
|
||||
ActionsDAGPtr before_where;
|
||||
String where_column_name;
|
||||
|
||||
@ -172,7 +172,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, ColumnPtr & colu
|
||||
|
||||
auto create_stream_getter = [&](bool stream_for_prefix)
|
||||
{
|
||||
return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer *
|
||||
return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * //-V1047
|
||||
{
|
||||
if (cache.count(ISerialization::getSubcolumnNameForStream(path)))
|
||||
return nullptr;
|
||||
|
||||
@ -38,6 +38,7 @@ MESSAGES_TO_RETRY = [
|
||||
"Coordination::Exception: Operation timeout",
|
||||
"Operation timed out",
|
||||
"ConnectionPoolWithFailover: Connection failed at try",
|
||||
"DB::Exception: New table appeared in database being dropped or detached. Try again"
|
||||
]
|
||||
|
||||
class Terminated(KeyboardInterrupt):
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
table 10 101
|
||||
a 0
|
||||
a 1
|
||||
a 2
|
||||
a 3
|
||||
a 4
|
||||
b 0
|
||||
b 1
|
||||
b 2
|
||||
b 3
|
||||
b 4
|
||||
@ -0,0 +1,30 @@
|
||||
select * from ( select 'table' as table, toInt64(10) as rows, toInt64(101) as elements union all select 'another table' as table, toInt64(0) as rows, toInt64(0) as elements ) where rows - elements <> 0;
|
||||
|
||||
SELECT
|
||||
label,
|
||||
number
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
'a' AS label,
|
||||
number
|
||||
FROM
|
||||
(
|
||||
SELECT number
|
||||
FROM numbers(10)
|
||||
)
|
||||
UNION ALL
|
||||
SELECT
|
||||
'b' AS label,
|
||||
number
|
||||
FROM
|
||||
(
|
||||
SELECT number
|
||||
FROM numbers(10)
|
||||
)
|
||||
)
|
||||
WHERE number IN
|
||||
(
|
||||
SELECT number
|
||||
FROM numbers(5)
|
||||
) order by label, number;
|
||||
Loading…
Reference in New Issue
Block a user