diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index da1f2122fc6..816164bef10 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -29,7 +29,6 @@ message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") # This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. set (REQUIRED_LLVM_LIBRARIES -LLVMOrcJIT LLVMExecutionEngine LLVMRuntimeDyld LLVMX86CodeGen diff --git a/contrib/libunwind b/contrib/libunwind index 8fe25d7dc70..a491c27b331 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 8fe25d7dc70f2a4ea38c3e5a33fa9d4199b67a5a +Subproject commit a491c27b33109a842d577c0f7ac5f5f218859181 diff --git a/contrib/llvm b/contrib/llvm index a7198805de6..e5751459412 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit a7198805de67374eb3fb4c6b89797fa2d1cd7e50 +Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1 diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 3a19a249f8e..fc73a0df0ee 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -374,6 +374,7 @@ function run_tests 01801_s3_cluster # Depends on LLVM JIT + 01072_nullable_jit 01852_jit_if 01865_jit_comparison_constant_result 01871_merge_tree_compile_expressions diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 8231caceca8..9628c512e83 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -44,7 +44,7 @@ parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated l parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.') parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') -parser.add_argument('--max-query-seconds', type=int, default=10, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') +parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.') parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') @@ -273,8 +273,14 @@ for query_index in queries_to_run: prewarm_id = f'{query_prefix}.prewarm0' try: - # Will also detect too long queries during warmup stage - res = c.execute(q, query_id = prewarm_id, settings = {'max_execution_time': args.max_query_seconds}) + # During the warmup runs, we will also: + # * detect queries that are exceedingly long, to fail fast, + # * collect profiler traces, which might be helpful for analyzing + # test coverage. We disable profiler for normal runs because + # it makes the results unstable. + res = c.execute(q, query_id = prewarm_id, + settings = {'max_execution_time': args.max_query_seconds, + 'query_profiler_real_time_period_ns': 10000000}) except clickhouse_driver.errors.Error as e: # Add query id to the exception to make debugging easier. e.args = (prewarm_id, *e.args) @@ -359,10 +365,11 @@ for query_index in queries_to_run: # For very short queries we have a special mode where we run them for at # least some time. The recommended lower bound of run time for "normal" # queries is about 0.1 s, and we run them about 10 times, giving the - # time per query per server of about one second. Use this value as a - # reference for "short" queries. + # time per query per server of about one second. Run "short" queries + # for longer time, because they have a high percentage of overhead and + # might give less stable results. if is_short[query_index]: - if server_seconds >= 2 * len(this_query_connections): + if server_seconds >= 8 * len(this_query_connections): break # Also limit the number of runs, so that we don't go crazy processing # the results -- 'eqmed.sql' is really suboptimal. diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 42490971127..503f2904461 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -446,6 +446,9 @@ if args.report == 'main': attrs[3] = f'style="background: {color_bad}"' else: attrs[3] = '' + # Just don't add the slightly unstable queries we don't consider + # errors. It's not clear what the user should do with them. + continue text += tableRow(r, attrs, anchor) @@ -553,12 +556,11 @@ if args.report == 'main': error_tests += unstable_partial_queries status = 'failure' - if unstable_queries: - message_array.append(str(unstable_queries) + ' unstable') - -# Disabled before fix. -# if very_unstable_queries: -# status = 'failure' + # Don't show mildly unstable queries, only the very unstable ones we + # treat as errors. + if very_unstable_queries: + status = 'failure' + message_array.append(str(very_unstable_queries) + ' unstable') error_tests += slow_average_tests if error_tests: diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index ef34c8d3804..8e4698b9bf8 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -101,6 +101,8 @@ For very large clusters, you can use different ZooKeeper clusters for different Replication is asynchronous and multi-master. `INSERT` queries (as well as `ALTER`) can be sent to any available server. Data is inserted on the server where the query is run, and then it is copied to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If part of the replicas are not available, the data is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. The number of threads performing background tasks for replicated tables can be set by [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) setting. +`ReplicatedMergeTree` engine uses a separate thread pool for replicated fetches. Size of the pool is limited by the [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) setting which can be tuned with a server restart. + By default, an INSERT query waits for confirmation of writing the data from only one replica. If the data was successfully written to only one replica and the server with this replica ceases to exist, the stored data will be lost. To enable getting confirmation of data writes from multiple replicas, use the `insert_quorum` option. Each block of data is written atomically. The INSERT query is divided into blocks up to `max_insert_block_size = 1048576` rows. In other words, if the `INSERT` query has less than 1048576 rows, it is made atomically. @@ -284,6 +286,7 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to **See Also** - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) [Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md index a52d82f21d0..96009c75af1 100644 --- a/docs/en/operations/configuration-files.md +++ b/docs/en/operations/configuration-files.md @@ -57,7 +57,7 @@ $ cat /etc/clickhouse-server/users.d/alice.xml ## YAML examples {#example} -Here you can see default config written in YAML: [config-example.yaml](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config-example.yaml). +Here you can see default config written in YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example). There are some differences between YAML and XML formats in terms of ClickHouse configurations. Here are some tips for writing a configuration in YAML format. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3c5efd79863..1a0202ad45a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2034,6 +2034,16 @@ Possible values: Default value: 16. +## background_fetches_pool_size {#background_fetches_pool_size} + +Sets the number of threads performing background fetches for [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. For production usage with frequent small insertions or slow ZooKeeper cluster is recomended to use default value. + +Possible values: + +- Any positive integer. + +Default value: 8. + ## always_fetch_merged_part {#always_fetch_merged_part} Prohibits data parts merging in [Replicated\*MergeTree](../../engines/table-engines/mergetree-family/replication.md)-engine tables. diff --git a/docs/en/sql-reference/functions/splitting-merging-functions.md b/docs/en/sql-reference/functions/splitting-merging-functions.md index d61896b6d98..2d384f1aa3c 100644 --- a/docs/en/sql-reference/functions/splitting-merging-functions.md +++ b/docs/en/sql-reference/functions/splitting-merging-functions.md @@ -13,7 +13,7 @@ Returns an array of selected substrings. Empty substrings may be selected if the **Syntax** ``` sql -splitByChar(, ) +splitByChar(separator, s) ``` **Arguments** @@ -29,12 +29,12 @@ Returns an array of selected substrings. Empty substrings may be selected when: - There are multiple consecutive separators; - The original string `s` is empty. -Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). **Example** ``` sql -SELECT splitByChar(',', '1,2,3,abcde') +SELECT splitByChar(',', '1,2,3,abcde'); ``` ``` text @@ -50,7 +50,7 @@ Splits a string into substrings separated by a string. It uses a constant string **Syntax** ``` sql -splitByString(, ) +splitByString(separator, s) ``` **Arguments** @@ -62,7 +62,7 @@ splitByString(, ) Returns an array of selected substrings. Empty substrings may be selected when: -Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). - A non-empty separator occurs at the beginning or end of the string; - There are multiple consecutive non-empty separators; @@ -71,7 +71,7 @@ Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-ref **Example** ``` sql -SELECT splitByString(', ', '1, 2 3, 4,5, abcde') +SELECT splitByString(', ', '1, 2 3, 4,5, abcde'); ``` ``` text @@ -81,7 +81,7 @@ SELECT splitByString(', ', '1, 2 3, 4,5, abcde') ``` ``` sql -SELECT splitByString('', 'abcde') +SELECT splitByString('', 'abcde'); ``` ``` text @@ -92,12 +92,12 @@ SELECT splitByString('', 'abcde') ## splitByRegexp(regexp, s) {#splitbyregexpseparator-s} -Splits a string into substrings separated by a regular expression. It uses a regular expression string `regexp` as the separator. If the `regexp` is empty, it will split the string s into an array of single characters. If no match is found for this regex expression, the string `s` won't be split. +Splits a string into substrings separated by a regular expression. It uses a regular expression string `regexp` as the separator. If the `regexp` is empty, it will split the string `s` into an array of single characters. If no match is found for this regular expression, the string `s` won't be split. **Syntax** ``` sql -splitByRegexp(, ) +splitByRegexp(regexp, s) ``` **Arguments** @@ -109,28 +109,36 @@ splitByRegexp(, ) Returns an array of selected substrings. Empty substrings may be selected when: - - A non-empty regular expression match occurs at the beginning or end of the string; - There are multiple consecutive non-empty regular expression matches; - The original string `s` is empty while the regular expression is not empty. -Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). + **Example** +Query: + ``` sql -SELECT splitByRegexp('\\d+', 'a12bc23de345f') +SELECT splitByRegexp('\\d+', 'a12bc23de345f'); ``` +Result: + ``` text ┌─splitByRegexp('\\d+', 'a12bc23de345f')─┐ │ ['a','bc','de','f'] │ └────────────────────────────────────────┘ ``` +Query: + ``` sql -SELECT splitByRegexp('', 'abcde') +SELECT splitByRegexp('', 'abcde'); ``` +Result: + ``` text ┌─splitByRegexp('', 'abcde')─┐ │ ['a','b','c','d','e'] │ @@ -149,7 +157,7 @@ Selects substrings of consecutive bytes from the ranges a-z and A-Z.Returns an a **Example** ``` sql -SELECT alphaTokens('abca1abc') +SELECT alphaTokens('abca1abc'); ``` ``` text diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 848adbee4da..cb92084695a 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -65,6 +65,8 @@ ClickHouse хранит метаинформацию о репликах в [Apa Репликация асинхронная, мульти-мастер. Запросы `INSERT` и `ALTER` можно направлять на любой доступный сервер. Данные вставятся на сервер, где выполнен запрос, а затем скопируются на остальные серверы. В связи с асинхронностью, только что вставленные данные появляются на остальных репликах с небольшой задержкой. Если часть реплик недоступна, данные на них запишутся тогда, когда они станут доступны. Если реплика доступна, то задержка составляет столько времени, сколько требуется для передачи блока сжатых данных по сети. Количество потоков для выполнения фоновых задач можно задать с помощью настройки [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size). +Движок `ReplicatedMergeTree` использует отдельный пул потоков для скачивания кусков данных. Размер пула ограничен настройкой [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size), которую можно указать при перезапуске сервера. + По умолчанию, запрос INSERT ждёт подтверждения записи только от одной реплики. Если данные были успешно записаны только на одну реплику, и сервер с этой репликой перестал существовать, то записанные данные будут потеряны. Вы можете включить подтверждение записи от нескольких реплик, используя настройку `insert_quorum`. Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до `max_insert_block_size = 1048576` строк. То есть, если в запросе `INSERT` менее 1048576 строк, то он делается атомарно. @@ -249,5 +251,6 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data **Смотрите также** - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md index 11a01d1e6d2..8b4b0da8f2b 100644 --- a/docs/ru/operations/configuration-files.md +++ b/docs/ru/operations/configuration-files.md @@ -6,9 +6,9 @@ toc_title: "Конфигурационные файлы" # Конфигурационные файлы {#configuration_files} -Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. +Основной конфигурационный файл сервера - `config.xml` или `config.yaml`. Он расположен в директории `/etc/clickhouse-server/`. -Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf` из директории `config.d` рядом с конфигом. +Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf`, а также `.yaml` (для файлов в формате YAML) из директории `config.d` рядом с конфигом. У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`. @@ -25,7 +25,7 @@ toc_title: "Конфигурационные файлы" В элементе `users_config` файла `config.xml` можно указать относительный путь к конфигурационному файлу с настройками пользователей, профилей и квот. Значение `users_config` по умолчанию — `users.xml`. Если `users_config` не указан, то настройки пользователей, профилей и квот можно задать непосредственно в `config.xml`. Настройки пользователя могут быть разделены в несколько отдельных файлов аналогичных `config.xml` и `config.d\`. Имя директории задаётся также как `users_config`. -Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`. +Имя директории задаётся так же, как имя файла в `users_config`, с подстановкой `.d` вместо `.xml`/`.yaml`. Директория `users.d` используется по умолчанию, также как `users.xml` используется для `users_config`. Например, можно иметь по отдельному конфигурационному файлу для каждого пользователя: @@ -52,3 +52,66 @@ $ cat /etc/clickhouse-server/users.d/alice.xml Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера. +## Примеры записи конфигурации на YAML {#example} + +Здесь можно рассмотреть пример реальной конфигурации записанной на YAML: [config.yaml.example](https://github.com/ClickHouse/ClickHouse/blob/master/programs/server/config.yaml.example). + +Между стандартами XML и YAML имеются различия, поэтому в этом разделе будут перечислены некоторые подсказки для написания конфигурации на YMAL. + +Для записи обычной пары ключ-значение следует использовать Scalar: +``` yaml +key: value +``` + +Для создания тега, содержащего подтеги следует использовать Map: +``` yaml +map_key: + key1: val1 + key2: val2 + key3: val3 +``` + +Для создания списка значений или подтегов, расположенных по определенному ключу, следует использовать Sequence: +``` yaml +seq_key: + - val1 + - val2 + - key1: val3 + - map: + key2: val4 + key3: val5 +``` + +В случае, усли необходимо объявить тег, аналогичный XML-атрибуту, необходимо задать скаляр, имеющий ключ с префиксом @ и заключенный в кавычки: + +``` yaml +map: + "@attr1": value1 + "@attr2": value2 + key: 123 +``` + +Из такой Map мы получим после конвертации: + +``` xml + + 123 + +``` + +Помимо Map, можно задавать атрибуты для Sequence: + +``` yaml +seq: + - "@attr1": value1 + - "@attr2": value2 + - 123 + - abc +``` + +Таким образом получая аналог следующей записи на XML: + +``` xml +123 +abc +``` diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 45def8c4b50..f84dd1e3ddb 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2043,6 +2043,16 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; Значение по умолчанию: 16. +## background_fetches_pool_size {#background_fetches_pool_size} + +Задает количество потоков для скачивания кусков данных для [реплицируемых](../../engines/table-engines/mergetree-family/replication.md) таблиц. Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе. Для использования в продакшене с частыми небольшими вставками или медленным кластером ZooKeeper рекомендуется использовать значение по умолчанию. + +Допустимые значения: + +- Положительное целое число. + +Значение по умолчанию: 8. + ## background_distributed_schedule_pool_size {#background_distributed_schedule_pool_size} Задает количество потоков для выполнения фоновых задач. Работает для таблиц с движком [Distributed](../../engines/table-engines/special/distributed.md). Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе. diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 4de487c03ad..8941ccc1691 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -18,37 +18,37 @@ toc_title: JSON Проверяет наличие поля с именем `name`. -Алиас: `simpleJSONHas`. +Синоним: `simpleJSONHas`. ## visitParamExtractUInt(params, name) {#visitparamextractuintparams-name} Пытается выделить число типа UInt64 из значения поля с именем `name`. Если поле строковое, пытается выделить число из начала строки. Если такого поля нет, или если оно есть, но содержит не число, то возвращает 0. -Алиас: `simpleJSONExtractUInt`. +Синоним: `simpleJSONExtractUInt`. ## visitParamExtractInt(params, name) {#visitparamextractintparams-name} Аналогично для Int64. -Алиас: `simpleJSONExtractInt`. +Синоним: `simpleJSONExtractInt`. ## visitParamExtractFloat(params, name) {#visitparamextractfloatparams-name} Аналогично для Float64. -Алиас: `simpleJSONExtractFloat`. +Синоним: `simpleJSONExtractFloat`. ## visitParamExtractBool(params, name) {#visitparamextractboolparams-name} Пытается выделить значение true/false. Результат — UInt8. -Алиас: `simpleJSONExtractBool`. +Синоним: `simpleJSONExtractBool`. ## visitParamExtractRaw(params, name) {#visitparamextractrawparams-name} Возвращает значение поля, включая разделители. -Алиас: `simpleJSONExtractRaw`. +Синоним: `simpleJSONExtractRaw`. Примеры: @@ -61,7 +61,7 @@ visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}'; Разбирает строку в двойных кавычках. У значения убирается экранирование. Если убрать экранированные символы не удалось, то возвращается пустая строка. -Алиас: `simpleJSONExtractString`. +Синоним: `simpleJSONExtractString`. Примеры: diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index b8d04982b91..5a0c540cf3a 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -14,7 +14,7 @@ separator должен быть константной строкой из ро **Синтаксис** ``` sql -splitByChar(, ) +splitByChar(separator, s) ``` **Аргументы** @@ -30,12 +30,12 @@ splitByChar(, ) - Задано несколько последовательных разделителей; - Исходная строка `s` пуста. -Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). **Пример** ``` sql -SELECT splitByChar(',', '1,2,3,abcde') +SELECT splitByChar(',', '1,2,3,abcde'); ``` ``` text @@ -67,12 +67,12 @@ splitByString(separator, s) - Задано несколько последовательных разделителей; - Исходная строка `s` пуста. -Тип: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). **Примеры** ``` sql -SELECT splitByString(', ', '1, 2 3, 4,5, abcde') +SELECT splitByString(', ', '1, 2 3, 4,5, abcde'); ``` ``` text @@ -82,7 +82,7 @@ SELECT splitByString(', ', '1, 2 3, 4,5, abcde') ``` ``` sql -SELECT splitByString('', 'abcde') +SELECT splitByString('', 'abcde'); ``` ``` text @@ -91,6 +91,60 @@ SELECT splitByString('', 'abcde') └────────────────────────────┘ ``` +## splitByRegexp(regexp, s) {#splitbyregexpseparator-s} + +Разбивает строку на подстроки, разделенные регулярным выражением. В качестве разделителя используется строка регулярного выражения `regexp`. Если `regexp` пустая, функция разделит строку `s` на массив одиночных символов. Если для регулярного выражения совпадения не найдено, строка `s` не будет разбита. + +**Синтаксис** + +``` sql +splitByRegexp(regexp, s) +``` + +**Аргументы** + +- `regexp` — регулярное выражение. Константа. [String](../data-types/string.md) или [FixedString](../data-types/fixedstring.md). +- `s` — разбиваемая строка. [String](../../sql-reference/data-types/string.md). + +**Возвращаемые значения** + +Возвращает массив выбранных подстрок. Пустая подстрока может быть возвращена, если: + +- Непустое совпадение с регулярным выражением происходит в начале или конце строки; +- Имеется несколько последовательных совпадений c непустым регулярным выражением; +- Исходная строка `s` пуста, а регулярное выражение не пустое. + +Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). + +**Примеры** + +Запрос: + +``` sql +SELECT splitByRegexp('\\d+', 'a12bc23de345f'); +``` + +Результат: + +``` text +┌─splitByRegexp('\\d+', 'a12bc23de345f')─┐ +│ ['a','bc','de','f'] │ +└────────────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT splitByRegexp('', 'abcde'); +``` + +Результат: + +``` text +┌─splitByRegexp('', 'abcde')─┐ +│ ['a','b','c','d','e'] │ +└────────────────────────────┘ +``` ## arrayStringConcat(arr\[, separator\]) {#arraystringconcatarr-separator} @@ -106,7 +160,7 @@ separator - необязательный параметр, константна **Пример:** ``` sql -SELECT alphaTokens('abca1abc') +SELECT alphaTokens('abca1abc'); ``` ``` text @@ -114,4 +168,3 @@ SELECT alphaTokens('abca1abc') │ ['abca','abc'] │ └─────────────────────────┘ ``` - diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 96d336673d0..8c187978106 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -983,7 +983,7 @@ int mainEntryClickHouseStop(int argc, char ** argv) desc.add_options() ("help,h", "produce help message") ("pid-path", po::value()->default_value("/var/run/clickhouse-server"), "directory for pid file") - ("force", po::value()->default_value(false), "Stop with KILL signal instead of TERM") + ("force", po::bool_switch(), "Stop with KILL signal instead of TERM") ; po::variables_map options; diff --git a/programs/server/config-example.yaml b/programs/server/config.yaml.example similarity index 99% rename from programs/server/config-example.yaml rename to programs/server/config.yaml.example index 3e275adf2fa..bebfd74ff58 100644 --- a/programs/server/config-example.yaml +++ b/programs/server/config.yaml.example @@ -1,4 +1,8 @@ -# NOTE: User and query level settings are set up in "users.xml" file. +# This is an example of a configuration file "config.xml" rewritten in YAML +# You can read this documentation for detailed information about YAML configuration: +# https://clickhouse.tech/docs/en/operations/configuration-files/ + +# NOTE: User and query level settings are set up in "users.yaml" file. # If you have accidentally specified user-level settings here, server won't start. # You can either move the settings to the right place inside "users.xml" file # or add skip_check_for_incorrect_settings: 1 here. diff --git a/programs/server/users.yaml.example b/programs/server/users.yaml.example new file mode 100644 index 00000000000..76aee04c19b --- /dev/null +++ b/programs/server/users.yaml.example @@ -0,0 +1,107 @@ +# Profiles of settings. +profiles: + # Default settings. + default: + # Maximum memory usage for processing single query, in bytes. + max_memory_usage: 10000000000 + + # How to choose between replicas during distributed query processing. + # random - choose random replica from set of replicas with minimum number of errors + # nearest_hostname - from set of replicas with minimum number of errors, choose replica + # with minimum number of different symbols between replica's hostname and local hostname (Hamming distance). + # in_order - first live replica is chosen in specified order. + # first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors. + load_balancing: random + + # Profile that allows only read queries. + readonly: + readonly: 1 + +# Users and ACL. +users: + # If user name was not specified, 'default' user is used. + default: + # Password could be specified in plaintext or in SHA256 (in hex format). + # + # If you want to specify password in plaintext (not recommended), place it in 'password' element. + # Example: password: qwerty + # Password could be empty. + # + # If you want to specify SHA256, place it in 'password_sha256_hex' element. + # Example: password_sha256_hex: 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 + # Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019). + # + # If you want to specify double SHA1, place it in 'password_double_sha1_hex' element. + # Example: password_double_sha1_hex: e395796d6546b1b65db9d665cd43f0e858dd4303 + # + # If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication, + # place its name in 'server' element inside 'ldap' element. + # Example: ldap: + # server: my_ldap_server + # + # If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config), + # place 'kerberos' element instead of 'password' (and similar) elements. + # The name part of the canonical principal name of the initiator must match the user name for authentication to succeed. + # You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests + # whose initiator's realm matches it. + # Example: kerberos: '' + # Example: kerberos: + # realm: EXAMPLE.COM + # + # How to generate decent password: + # Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + # In first line will be password and in second - corresponding SHA256. + # + # How to generate double SHA1: + # Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' + # In first line will be password and in second - corresponding double SHA1. + + password: '' + + # List of networks with open access. + # + # To open access from everywhere, specify: + # - ip: '::/0' + # + # To open access only from localhost, specify: + # - ip: '::1' + # - ip: 127.0.0.1 + # + # Each element of list has one of the following forms: + # ip: IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0 + # 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::. + # host: Hostname. Example: server01.yandex.ru. + # To check access, DNS query is performed, and all received addresses compared to peer address. + # host_regexp: Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ + # To check access, DNS PTR query is performed for peer address and then regexp is applied. + # Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address. + # Strongly recommended that regexp is ends with $ and take all expression in '' + # All results of DNS requests are cached till server restart. + + networks: + ip: '::/0' + + # Settings profile for user. + profile: default + + # Quota for user. + quota: default + + # User can create other users and grant rights to them. + # access_management: 1 + +# Quotas. +quotas: + # Name of quota. + default: + # Limits for time interval. You could specify many intervals with different limits. + interval: + # Length of interval. + duration: 3600 + + # No limits. Just calculate resource usage for time interval. + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 diff --git a/src/Common/ya.make b/src/Common/ya.make index dde1e6ae013..6bc15da2cdb 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -33,6 +33,7 @@ SRCS( Config/AbstractConfigurationComparison.cpp Config/ConfigProcessor.cpp Config/ConfigReloader.cpp + Config/YAMLParser.cpp Config/configReadClient.cpp CurrentMemoryTracker.cpp CurrentMetrics.cpp diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 325e887073c..1a347394012 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -54,6 +54,7 @@ class DiskS3::AwsS3KeyKeeper : public std::list & chunk); private: /// limit for one DeleteObject request @@ -74,6 +75,19 @@ void DiskS3::AwsS3KeyKeeper::addKey(const String & key) back().push_back(obj); } +String DiskS3::AwsS3KeyKeeper::getChunkKeys(const Aws::Vector & chunk) +{ + String res; + for (const auto & obj : chunk) + { + const auto & key = obj.GetKey(); + if (!res.empty()) + res.append(", "); + res.append(key.c_str(), key.size()); + } + return res; +} + String getRandomName() { std::uniform_int_distribution distribution('a', 'z'); @@ -794,6 +808,8 @@ void DiskS3::removeAws(const AwsS3KeyKeeper & keys) for (const auto & chunk : keys) { + LOG_DEBUG(log, "Remove AWS keys {}", AwsS3KeyKeeper::getChunkKeys(chunk)); + Aws::S3::Model::Delete delkeys; delkeys.SetObjects(chunk); diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index ab8e1cfc0b2..68eed88e59c 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -521,6 +521,70 @@ ColumnPtr FunctionAnyArityLogical::executeImpl( return basicExecuteImpl(std::move(args_in), input_rows_count); } +template +ColumnPtr FunctionAnyArityLogical::getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const +{ + /** Try to perform optimization for saturable functions (AndFunction, OrFunction) in case some arguments are + * constants. + * If function is not saturable (XorFunction) we cannot perform such optimization. + * If function is AndFunction and in arguments there is constant false, result is false. + * If function is OrFunction and in arguments there is constant true, result is true. + */ + if constexpr (!Impl::isSaturable()) + return nullptr; + + bool has_true_constant = false; + bool has_false_constant = false; + + for (const auto & argument : arguments) + { + ColumnPtr column = argument.column; + + if (!column || !isColumnConst(*column)) + continue; + + DataTypePtr non_nullable_type = removeNullable(argument.type); + TypeIndex data_type_index = non_nullable_type->getTypeId(); + + if (!isNativeNumber(data_type_index)) + continue; + + const ColumnConst * const_column = static_cast(column.get()); + + Field constant_field_value = const_column->getField(); + if (constant_field_value.isNull()) + continue; + + auto field_type = constant_field_value.getType(); + + bool constant_value_bool = false; + + if (field_type == Field::Types::Float64) + constant_value_bool = static_cast(constant_field_value.get()); + else if (field_type == Field::Types::Int64) + constant_value_bool = static_cast(constant_field_value.get()); + else if (field_type == Field::Types::UInt64) + constant_value_bool = static_cast(constant_field_value.get()); + + has_true_constant = has_true_constant || constant_value_bool; + has_false_constant = has_false_constant || !constant_value_bool; + } + + ColumnPtr result_column; + + if constexpr (std::is_same_v) + { + if (has_false_constant) + result_type->createColumnConst(0, static_cast(false)); + } + else if constexpr (std::is_same_v) + { + if (has_true_constant) + result_type->createColumnConst(0, static_cast(true)); + } + + return result_column; +} template struct UnaryOperationImpl diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index acc34a40d00..264eeeef0fe 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -155,7 +155,9 @@ public: /// Get result types by argument types. If the function does not apply to these arguments, throw an exception. DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override; + + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override; #if USE_EMBEDDED_COMPILER bool isCompilableImpl(const DataTypes &) const override { return useDefaultImplementationForNulls(); } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 7542451a81a..0da55343b9d 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -155,12 +155,13 @@ public: */ virtual bool isSuitableForConstantFolding() const { return true; } - /** Some functions like ignore(...) or toTypeName(...) always return constant result which doesn't depend on arguments. - * In this case we can calculate result and assume that it's constant in stream header. - * There is no need to implement function if it has zero arguments. - * Must return ColumnConst with single row or nullptr. + /** If function isSuitableForConstantFolding then, this method will be called during query analyzis + * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can + * return they result based on some constant arguments. + * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants, + * useDefaultImplementationForLowCardinality are not applied. */ - virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*columns*/) const { return nullptr; } + virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; } /** Function is called "injective" if it returns different result for different values of arguments. * Example: hex, negate, tuple... @@ -300,7 +301,7 @@ protected: return getReturnTypeImpl(data_types); } - /** If useDefaultImplementationForNulls() is true, than change arguments for getReturnType() and build(): + /** If useDefaultImplementationForNulls() is true, then change arguments for getReturnType() and build(): * if some of arguments are Nullable(Nothing) then don't call getReturnType(), call build() with return_type = Nullable(Nothing), * if some of arguments are Nullable, then: * - Nullable types are substituted with nested types for getReturnType() function @@ -310,7 +311,7 @@ protected: */ virtual bool useDefaultImplementationForNulls() const { return true; } - /** If useDefaultImplementationForNulls() is true, than change arguments for getReturnType() and build(). + /** If useDefaultImplementationForNulls() is true, then change arguments for getReturnType() and build(). * If function arguments has low cardinality types, convert them to ordinary types. * getReturnType returns ColumnLowCardinality if at least one argument type is ColumnLowCardinality. */ @@ -377,7 +378,7 @@ public: /// Properties from IFunctionBase (see IFunction.h) virtual bool isSuitableForConstantFolding() const { return true; } - virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; } + virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*result_type*/) const { return nullptr; } virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; } virtual bool isDeterministic() const { return true; } virtual bool isDeterministicInScopeOfQuery() const { return true; } diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index b213a459103..6a865af0dd3 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -66,9 +66,10 @@ public: } bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments_) const override + + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments_, const DataTypePtr & result_type_) const override { - return function->getResultIfAlwaysReturnsConstantAndHasArguments(arguments_); + return function->getConstantResultForNonConstArguments(arguments_, result_type_); } bool isStateful() const override { return function->isStateful(); } diff --git a/src/Functions/defaultValueOfArgumentType.cpp b/src/Functions/defaultValueOfArgumentType.cpp index 957d829039d..c0abe1b63d9 100644 --- a/src/Functions/defaultValueOfArgumentType.cpp +++ b/src/Functions/defaultValueOfArgumentType.cpp @@ -42,7 +42,7 @@ public: return type.createColumnConst(input_rows_count, type.getDefault()); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override { const IDataType & type = *arguments[0].type; return type.createColumnConst(1, type.getDefault()); diff --git a/src/Functions/getSizeOfEnumType.cpp b/src/Functions/getSizeOfEnumType.cpp index 7db11d41d0f..504c088a737 100644 --- a/src/Functions/getSizeOfEnumType.cpp +++ b/src/Functions/getSizeOfEnumType.cpp @@ -51,17 +51,24 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - return getResultIfAlwaysReturnsConstantAndHasArguments(arguments)->cloneResized(input_rows_count); + return getSizeOfEnumType(arguments[0].type, input_rows_count); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override { - if (const auto * type8 = checkAndGetDataType(arguments[0].type.get())) - return DataTypeUInt8().createColumnConst(1, type8->getValues().size()); - else if (const auto * type16 = checkAndGetDataType(arguments[0].type.get())) - return DataTypeUInt16().createColumnConst(1, type16->getValues().size()); + return getSizeOfEnumType(arguments[0].type, 1); + } + +private: + + ColumnPtr getSizeOfEnumType(const DataTypePtr & data_type, size_t input_rows_count) const + { + if (const auto * type8 = checkAndGetDataType(data_type.get())) + return DataTypeUInt8().createColumnConst(input_rows_count, type8->getValues().size()); + else if (const auto * type16 = checkAndGetDataType(data_type.get())) + return DataTypeUInt16().createColumnConst(input_rows_count, type16->getValues().size()); else - throw Exception("The argument for function " + getName() + " must be Enum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The argument for function {} must be Enum", getName()); } }; diff --git a/src/Functions/ignore.cpp b/src/Functions/ignore.cpp index 45079639835..176bc9afee2 100644 --- a/src/Functions/ignore.cpp +++ b/src/Functions/ignore.cpp @@ -49,11 +49,6 @@ public: { return DataTypeUInt8().createColumnConst(input_rows_count, 0u); } - - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override - { - return DataTypeUInt8().createColumnConst(1, 0u); - } }; } diff --git a/src/Functions/indexHint.cpp b/src/Functions/indexHint.cpp index d2a9a0abb8c..f3c856c38ce 100644 --- a/src/Functions/indexHint.cpp +++ b/src/Functions/indexHint.cpp @@ -55,10 +55,6 @@ public: return DataTypeUInt8().createColumnConst(input_rows_count, 1u); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override - { - return DataTypeUInt8().createColumnConst(1, 1u); - } }; diff --git a/src/Functions/timezoneOf.cpp b/src/Functions/timezoneOf.cpp index 0e6fa1bb213..3dc72424a4f 100644 --- a/src/Functions/timezoneOf.cpp +++ b/src/Functions/timezoneOf.cpp @@ -23,66 +23,12 @@ namespace /** timezoneOf(x) - get the name of the timezone of DateTime data type. * Example: Europe/Moscow. */ -class ExecutableFunctionTimezoneOf : public IExecutableFunction +class FunctionTimezoneOf : public IFunction { public: static constexpr auto name = "timezoneOf"; String getName() const override { return name; } - - bool useDefaultImplementationForNulls() const override { return false; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } - - /// Execute the function on the columns. - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - DataTypePtr type_no_nullable = removeNullable(arguments[0].type); - - return DataTypeString().createColumnConst(input_rows_count, - dynamic_cast(*type_no_nullable).getTimeZone().getTimeZone()); - } -}; - - -class BaseFunctionTimezoneOf : public IFunctionBase -{ -public: - BaseFunctionTimezoneOf(DataTypes argument_types_, DataTypePtr return_type_) - : argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {} - - static constexpr auto name = "timezoneOf"; - String getName() const override { return name; } - - bool isDeterministic() const override { return true; } - bool isDeterministicInScopeOfQuery() const override { return true; } - - const DataTypes & getArgumentTypes() const override { return argument_types; } - const DataTypePtr & getResultType() const override { return return_type; } - - ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override - { - return std::make_unique(); - } - - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override - { - DataTypePtr type_no_nullable = removeNullable(arguments[0].type); - - return DataTypeString().createColumnConst(1, - dynamic_cast(*type_no_nullable).getTimeZone().getTimeZone()); - } - -private: - DataTypes argument_types; - DataTypePtr return_type; -}; - - -class FunctionTimezoneOfBuilder : public IFunctionOverloadResolver -{ -public: - static constexpr auto name = "timezoneOf"; - String getName() const override { return name; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } + static FunctionPtr create(ContextPtr) { return std::make_unique(); } size_t getNumberOfArguments() const override { return 1; } @@ -96,21 +42,32 @@ public: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad argument for function {}, should be DateTime or DateTime64", name); } - FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override - { - return std::make_unique(DataTypes{arguments[0].type}, return_type); - } - bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + DataTypePtr type_no_nullable = removeNullable(arguments[0].type); + + return DataTypeString().createColumnConst(input_rows_count, + dynamic_cast(*type_no_nullable).getTimeZone().getTimeZone()); + } + + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override + { + DataTypePtr type_no_nullable = removeNullable(arguments[0].type); + + return DataTypeString().createColumnConst(1, + dynamic_cast(*type_no_nullable).getTimeZone().getTimeZone()); + } }; } void registerFunctionTimezoneOf(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); factory.registerAlias("timeZoneOf", "timezoneOf"); } diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index 0b73fe772d1..d64fa12604e 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -41,7 +41,7 @@ public: return DataTypeString().createColumnConst(input_rows_count, arguments[0].column->getName()); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & arguments) const override + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override { return DataTypeString().createColumnConst(1, arguments[0].type->createColumn()->getName()); } diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 4c7b00e74ed..3c733fb3ea7 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -12,85 +12,55 @@ namespace /** toTypeName(x) - get the type name * Returns name of IDataType instance (name of data type). */ -class ExecutableFunctionToTypeName : public IExecutableFunction +class FunctionToTypeName : public IFunction { public: + static constexpr auto name = "toTypeName"; - String getName() const override { return name; } + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } - /// Execute the function on the columns. + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { return DataTypeString().createColumnConst(input_rows_count, arguments[0].type->getName()); } -}; - -class BaseFunctionToTypeName : public IFunctionBase -{ -public: - BaseFunctionToTypeName(DataTypes argument_types_, DataTypePtr return_type_) - : argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {} - - static constexpr auto name = "toTypeName"; - String getName() const override { return name; } - - bool isDeterministic() const override { return true; } - bool isDeterministicInScopeOfQuery() const override { return true; } - - const DataTypes & getArgumentTypes() const override { return argument_types; } - const DataTypePtr & getResultType() const override { return return_type; } - - ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName &) const override + ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override { - return std::make_unique(); + return DataTypeString().createColumnConst(1, arguments[0].type->getName()); } - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName &) const override - { - return DataTypeString().createColumnConst(1, argument_types.at(0)->getName()); - } - -private: - DataTypes argument_types; - DataTypePtr return_type; -}; - - -class FunctionToTypeNameBuilder : public IFunctionOverloadResolver -{ -public: - static constexpr auto name = "toTypeName"; - String getName() const override { return name; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } - - size_t getNumberOfArguments() const override { return 1; } - - DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } - - FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override - { - DataTypes types; - types.reserve(arguments.size()); - for (const auto & elem : arguments) - types.emplace_back(elem.type); - - return std::make_unique(types, return_type); - } - - bool useDefaultImplementationForNulls() const override { return false; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } + }; } void registerFunctionToTypeName(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index d1c10e1c037..35251c7b9b7 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -205,34 +205,31 @@ const ActionsDAG::Node & ActionsDAG::addFunction( node.function = node.function_base->prepare(arguments); /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function. - if (all_const && node.function_base->isSuitableForConstantFolding()) + if (node.function_base->isSuitableForConstantFolding()) { - size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size(); - auto col = node.function->execute(arguments, node.result_type, num_rows, true); + ColumnPtr column; + + if (all_const) + { + size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size(); + column = node.function->execute(arguments, node.result_type, num_rows, true); + } + else + { + column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type); + } /// If the result is not a constant, just in case, we will consider the result as unknown. - if (isColumnConst(*col)) + if (column && isColumnConst(*column)) { /// All constant (literal) columns in block are added with size 1. /// But if there was no columns in block before executing a function, the result has size 0. /// Change the size to 1. - if (col->empty()) - col = col->cloneResized(1); + if (column->empty()) + column = column->cloneResized(1); - node.column = std::move(col); - } - } - - /// Some functions like ignore(), indexHint() or getTypeName() always return constant result even if arguments are not constant. - /// We can't do constant folding, but can specify in sample block that function result is constant to avoid - /// unnecessary materialization. - if (!node.column && node.function_base->isSuitableForConstantFolding()) - { - if (auto col = node.function_base->getResultIfAlwaysReturnsConstantAndHasArguments(arguments)) - { - node.column = std::move(col); - node.allow_constant_folding = false; + node.column = std::move(column); } } @@ -408,13 +405,10 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) for (auto & node : nodes) { - /// We cannot remove function with side effects even if it returns constant (e.g. ignore(...)). - bool prevent_constant_folding = node.column && isColumnConst(*node.column) && !node.allow_constant_folding; /// We cannot remove arrayJoin because it changes the number of rows. bool is_array_join = node.type == ActionType::ARRAY_JOIN; - bool must_keep_node = is_array_join || prevent_constant_folding; - if (must_keep_node && visited_nodes.count(&node) == 0) + if (is_array_join && visited_nodes.count(&node) == 0) { visited_nodes.insert(&node); stack.push(&node); @@ -429,7 +423,7 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) auto * node = stack.top(); stack.pop(); - if (!node->children.empty() && node->column && isColumnConst(*node->column) && node->allow_constant_folding) + if (!node->children.empty() && node->column && isColumnConst(*node->column)) { /// Constant folding. node->type = ActionsDAG::ActionType::COLUMN; @@ -1520,7 +1514,7 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & auto res = split(split_nodes); /// Do not remove array joined columns if they are not used. - res.first->project_input = false; + /// res.first->project_input = false; return res; } @@ -1533,7 +1527,9 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co column_name, dumpDAG()); std::unordered_set split_nodes = {node}; - return split(split_nodes); + auto res = split(split_nodes); + res.second->project_input = project_input; + return res; } namespace diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index cc1d9a0e6ac..9bea1d1c040 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -88,9 +88,6 @@ public: /// For COLUMN node and propagated constants. ColumnPtr column; - /// Some functions like `ignore()` always return constant but can't be replaced by constant it. - /// We calculate such constants in order to avoid unnecessary materialization, but prohibit it's folding. - bool allow_constant_folding = true; void toTree(JSONBuilder::JSONMap & map) const; }; diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index be693fdc3b9..113a4abb808 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -312,7 +312,7 @@ static FunctionBasePtr compile( static bool isCompilableConstant(const ActionsDAG::Node & node) { - return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding; + return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type); } static bool isCompilableFunction(const ActionsDAG::Node & node) @@ -334,16 +334,11 @@ static bool isCompilableFunction(const ActionsDAG::Node & node) return function.isCompilable(); } -static bool isCompilableInput(const ActionsDAG::Node & node) -{ - return node.type == ActionsDAG::ActionType::INPUT || node.type == ActionsDAG::ActionType::ALIAS; -} - static CompileDAG getCompilableDAG( const ActionsDAG::Node * root, ActionsDAG::NodeRawConstPtrs & children) { - /// Extract CompileDAG from root actions dag node, it is important that each root child is compilable. + /// Extract CompileDAG from root actions dag node. CompileDAG dag; @@ -363,6 +358,32 @@ static CompileDAG getCompilableDAG( auto & frame = stack.top(); const auto * node = frame.node; + bool is_compilable_constant = isCompilableConstant(*node); + bool is_compilable_function = isCompilableFunction(*node); + + if (!is_compilable_function || is_compilable_constant) + { + CompileDAG::Node compile_node; + compile_node.function = node->function_base; + compile_node.result_type = node->result_type; + + if (is_compilable_constant) + { + compile_node.type = CompileDAG::CompileType::CONSTANT; + compile_node.column = node->column; + } + else + { + compile_node.type = CompileDAG::CompileType::INPUT; + children.emplace_back(node); + } + + visited_node_to_compile_dag_position[node] = dag.getNodesCount(); + dag.addNode(std::move(compile_node)); + stack.pop(); + continue; + } + while (frame.next_child_to_visit < node->children.size()) { const auto & child = node->children[frame.next_child_to_visit]; @@ -382,26 +403,15 @@ static CompileDAG getCompilableDAG( if (!all_children_visited) continue; + /// Here we process only functions that are not compiled constants + CompileDAG::Node compile_node; compile_node.function = node->function_base; compile_node.result_type = node->result_type; + compile_node.type = CompileDAG::CompileType::FUNCTION; - if (isCompilableConstant(*node)) - { - compile_node.type = CompileDAG::CompileType::CONSTANT; - compile_node.column = node->column; - } - else if (node->type == ActionsDAG::ActionType::FUNCTION) - { - compile_node.type = CompileDAG::CompileType::FUNCTION; - for (const auto * child : node->children) - compile_node.arguments.push_back(visited_node_to_compile_dag_position[child]); - } - else - { - compile_node.type = CompileDAG::CompileType::INPUT; - children.emplace_back(node); - } + for (const auto * child : node->children) + compile_node.arguments.push_back(visited_node_to_compile_dag_position[child]); visited_node_to_compile_dag_position[node] = dag.getNodesCount(); @@ -417,8 +427,8 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) struct Data { bool is_compilable_in_isolation = false; - bool all_children_compilable = false; bool all_parents_compilable = true; + size_t compilable_children_size = 0; size_t children_size = 0; }; @@ -428,7 +438,7 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) for (const auto & node : nodes) { - bool node_is_compilable_in_isolation = isCompilableConstant(node) || isCompilableFunction(node) || isCompilableInput(node); + bool node_is_compilable_in_isolation = isCompilableFunction(node) && !isCompilableConstant(node); node_to_data[&node].is_compilable_in_isolation = node_is_compilable_in_isolation; } @@ -441,8 +451,7 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) std::stack stack; std::unordered_set visited_nodes; - /** Algorithm is to iterate over each node in ActionsDAG, and update node compilable status. - * Node is compilable if all its children are compilable and node is also compilable. + /** Algorithm is to iterate over each node in ActionsDAG, and update node compilable_children_size. * After this procedure data for each node is initialized. */ @@ -479,14 +488,18 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) auto & current_node_data = node_to_data[current_node]; - current_node_data.all_children_compilable = true; - if (current_node_data.is_compilable_in_isolation) { for (const auto * child : current_node->children) { - current_node_data.all_children_compilable &= node_to_data[child].is_compilable_in_isolation; - current_node_data.all_children_compilable &= node_to_data[child].all_children_compilable; + auto & child_data = node_to_data[child]; + + if (child_data.is_compilable_in_isolation) + { + current_node_data.compilable_children_size += child_data.compilable_children_size; + current_node_data.compilable_children_size += 1; + } + current_node_data.children_size += node_to_data[child].children_size; } @@ -501,10 +514,10 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) for (const auto & node : nodes) { auto & node_data = node_to_data[&node]; - bool is_compilable = node_data.is_compilable_in_isolation && node_data.all_children_compilable; + bool node_is_valid_for_compilation = node_data.is_compilable_in_isolation && node_data.compilable_children_size > 0; for (const auto & child : node.children) - node_to_data[child].all_parents_compilable &= is_compilable; + node_to_data[child].all_parents_compilable &= node_is_valid_for_compilation; } for (const auto & node : index) @@ -519,11 +532,10 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression) { auto & node_data = node_to_data[&node]; - bool node_is_valid_for_compilation = !isCompilableConstant(node) && node.children.size() > 1; - bool can_be_compiled = node_data.is_compilable_in_isolation && node_data.all_children_compilable && node_is_valid_for_compilation; + bool node_is_valid_for_compilation = node_data.is_compilable_in_isolation && node_data.compilable_children_size > 0; /// If all parents are compilable then this node should not be standalone compiled - bool should_compile = can_be_compiled && !node_data.all_parents_compilable; + bool should_compile = node_is_valid_for_compilation && !node_data.all_parents_compilable; if (!should_compile) continue; diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 88ddfefef63..7e58cccf13a 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -302,7 +302,7 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - auto stream = executeQuery(select_query, getContext()->getGlobalContext(), true).getInputStream(); + auto stream = executeQuery(select_query, getContext(), true).getInputStream(); Block res = stream->read(); if (res && stream->read()) diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index 7403dd0a12a..418e9b4e2e7 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include namespace DB { @@ -47,6 +49,28 @@ QueryPipelinePtr UnionStep::updatePipeline(QueryPipelines pipelines, const Build return pipeline; } + for (auto & cur_pipeline : pipelines) + { +#if !defined(NDEBUG) + assertCompatibleHeader(cur_pipeline->getHeader(), getOutputStream().header, "UnionStep"); +#endif + /// Headers for union must be equal. + /// But, just in case, convert it to the same header if not. + if (!isCompatibleHeader(cur_pipeline->getHeader(), getOutputStream().header)) + { + auto converting_dag = ActionsDAG::makeConvertingActions( + cur_pipeline->getHeader().getColumnsWithTypeAndName(), + getOutputStream().header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + auto converting_actions = std::make_shared(std::move(converting_dag)); + cur_pipeline->addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared(cur_header, converting_actions); + }); + } + } + *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), max_threads); processors = collector.detachProcessors(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 0c3b0998a55..f960c04abbe 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -268,8 +268,7 @@ void ReplicatedMergeTreeQueue::removeCoveredPartsFromMutations(const String & pa bool some_mutations_are_probably_done = false; - auto from_it = in_partition->second.lower_bound(part_info.getDataVersion()); - for (auto it = from_it; it != in_partition->second.end(); ++it) + for (auto it = in_partition->second.begin(); it != in_partition->second.end(); ++it) { MutationStatus & status = *it->second; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3adf3026b23..1782d3a6949 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4999,7 +4999,10 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St auto zookeeper = getZooKeeper(); delimiting_block_lock = allocateBlockNumber(partition_id, zookeeper); right = delimiting_block_lock->getNumber(); - mutation_version = queue.getCurrentMutationVersion(partition_id, right); + /// Make sure we cover all parts in drop range. + /// There might be parts with mutation version greater than current block number + /// if some part mutation has been assigned after block number allocation, but before creation of DROP_RANGE entry. + mutation_version = MergeTreePartInfo::MAX_BLOCK_NUMBER; } if (for_replace_range) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c1878d4c52d..177d28ce965 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -38,6 +38,7 @@ MESSAGES_TO_RETRY = [ "Coordination::Exception: Operation timeout", "Operation timed out", "ConnectionPoolWithFailover: Connection failed at try", + "DB::Exception: New table appeared in database being dropped or detached. Try again" ] class Terminated(KeyboardInterrupt): diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 6ca9e3eb65a..f609e6ddecc 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/arithmetic.xml b/tests/performance/arithmetic.xml index bf5e7662e37..7c3734a900a 100644 --- a/tests/performance/arithmetic.xml +++ b/tests/performance/arithmetic.xml @@ -1,4 +1,4 @@ - + 30000000000 diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index cf92b51f545..ca280ce28ad 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/avg_weighted.xml b/tests/performance/avg_weighted.xml index 2476011e6a9..fa39dd552cd 100644 --- a/tests/performance/avg_weighted.xml +++ b/tests/performance/avg_weighted.xml @@ -1,4 +1,4 @@ - + 1 1 diff --git a/tests/performance/bounding_ratio.xml b/tests/performance/bounding_ratio.xml index e430136b624..e3a15f90013 100644 --- a/tests/performance/bounding_ratio.xml +++ b/tests/performance/bounding_ratio.xml @@ -1,4 +1,4 @@ - + SELECT boundingRatio(number, number) FROM numbers(100000000) SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(100000000) diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index b282bcc268f..b31e0eafdd7 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,5 +1,4 @@ - - + 1 diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 662df80ae70..caefaba3725 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/collations.xml b/tests/performance/collations.xml index 52ccede3798..17b2d36b7e3 100644 --- a/tests/performance/collations.xml +++ b/tests/performance/collations.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/date_time_64.xml b/tests/performance/date_time_64.xml index fd883416a33..838aba34d87 100644 --- a/tests/performance/date_time_64.xml +++ b/tests/performance/date_time_64.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index c2eb42d3318..0c3d85f9659 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -1,4 +1,4 @@ - + datetime_transform diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 3fc1408d7e4..c8f7c8ed4e6 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -1,4 +1,4 @@ - + 35G diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 6dd38b6a06a..f087d0938c1 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,4 +1,4 @@ - + 15G diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml index 3f01449ed99..7983c058928 100644 --- a/tests/performance/direct_dictionary.xml +++ b/tests/performance/direct_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_direct_dictionary_source_table ( diff --git a/tests/performance/encodeXMLComponent.xml b/tests/performance/encodeXMLComponent.xml index 45241941ac3..7379a0ff89e 100644 --- a/tests/performance/encodeXMLComponent.xml +++ b/tests/performance/encodeXMLComponent.xml @@ -1,7 +1,7 @@ - test.hits + hits_100m_single - SELECT count() FROM test.hits WHERE NOT ignore(encodeXMLComponent(URL)) + SELECT count() FROM hits_100m_single WHERE NOT ignore(encodeXMLComponent(URL)) diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index a80631db541..243f90a8e59 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_flat_dictionary_source_table ( diff --git a/tests/performance/float_parsing.xml b/tests/performance/float_parsing.xml index eb8577bd127..33ab8ba6f10 100644 --- a/tests/performance/float_parsing.xml +++ b/tests/performance/float_parsing.xml @@ -1,4 +1,4 @@ - + expr diff --git a/tests/performance/fuzz_bits.xml b/tests/performance/fuzz_bits.xml index 87064e520c2..2679977cb1d 100644 --- a/tests/performance/fuzz_bits.xml +++ b/tests/performance/fuzz_bits.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/general_purpose_hashes.xml b/tests/performance/general_purpose_hashes.xml index f34554360cf..bd2fa9674f6 100644 --- a/tests/performance/general_purpose_hashes.xml +++ b/tests/performance/general_purpose_hashes.xml @@ -1,4 +1,4 @@ - + gp_hash_func diff --git a/tests/performance/general_purpose_hashes_on_UUID.xml b/tests/performance/general_purpose_hashes_on_UUID.xml index 3694e0a38bd..ccff0619c6e 100644 --- a/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/tests/performance/general_purpose_hashes_on_UUID.xml @@ -1,4 +1,4 @@ - + hash_func diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml index 0339a8c19e8..bc49a7de1bd 100644 --- a/tests/performance/generate_table_function.xml +++ b/tests/performance/generate_table_function.xml @@ -1,4 +1,4 @@ - + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000); diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index 26164b4f888..8dfb33ca53d 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_hashed_dictionary_source_table ( diff --git a/tests/performance/int_parsing.xml b/tests/performance/int_parsing.xml index 32f904331ce..3b8620e46c3 100644 --- a/tests/performance/int_parsing.xml +++ b/tests/performance/int_parsing.xml @@ -1,4 +1,4 @@ - + hits_100m_single hits_10m_single diff --git a/tests/performance/jit_small_requests.xml b/tests/performance/jit_small_requests.xml index d8f917fb9af..c9abec0926b 100644 --- a/tests/performance/jit_small_requests.xml +++ b/tests/performance/jit_small_requests.xml @@ -1,4 +1,4 @@ - + WITH bitXor(number, 0x4CF2D2BAAE6DA887) AS x0, diff --git a/tests/performance/json_extract_rapidjson.xml b/tests/performance/json_extract_rapidjson.xml index 2a34acfe10f..f8d40c1e58d 100644 --- a/tests/performance/json_extract_rapidjson.xml +++ b/tests/performance/json_extract_rapidjson.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/json_extract_simdjson.xml b/tests/performance/json_extract_simdjson.xml index 9ec3613d5e8..87b1eaac73e 100644 --- a/tests/performance/json_extract_simdjson.xml +++ b/tests/performance/json_extract_simdjson.xml @@ -1,4 +1,4 @@ - + json diff --git a/tests/performance/math.xml b/tests/performance/math.xml index 35250351683..cfa65809197 100644 --- a/tests/performance/math.xml +++ b/tests/performance/math.xml @@ -1,4 +1,4 @@ - + func_slow diff --git a/tests/performance/merge_tree_many_partitions_2.xml b/tests/performance/merge_tree_many_partitions_2.xml index 0b8a4650835..6799153ed65 100644 --- a/tests/performance/merge_tree_many_partitions_2.xml +++ b/tests/performance/merge_tree_many_partitions_2.xml @@ -1,4 +1,4 @@ - + CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index 77b39da8e92..8569a9431c1 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -1,4 +1,4 @@ - + format diff --git a/tests/performance/optimized_select_final.xml b/tests/performance/optimized_select_final.xml index d70fccc1330..46b408d0cb4 100644 --- a/tests/performance/optimized_select_final.xml +++ b/tests/performance/optimized_select_final.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/parallel_mv.xml b/tests/performance/parallel_mv.xml index e6635a74b96..1e9b92c54df 100644 --- a/tests/performance/parallel_mv.xml +++ b/tests/performance/parallel_mv.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml index 31c24eb006f..a22affb8e63 100644 --- a/tests/performance/point_in_polygon.xml +++ b/tests/performance/point_in_polygon.xml @@ -1,4 +1,4 @@ - + + SELECT sumIf(1, 0) FROM numbers(1000000000) + SELECT sumIf(1, 1) FROM numbers(1000000000) diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 82673067314..1fb7cf967f3 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -85,7 +85,8 @@ (81277, 26395), (57176, 69906), (55080, 20726), (65843, 9495), (27703, 24879), (60594, 53447), (72896, 95513), (87884, 50244), (25281, 16427), (14122, 6399), (65765, 45679), (38160, 38650), (76820, 91595), (74663, 37143), (35220, 16865), (54664, 88840), (7671, 19781), (7255, 87276), (92182, 47030), (98922, 49381), (21954, 20742), (54674, 19730), (74694, 81672), (31070, 97794), (22759, 3530), (6085, 72621), (17392, 26996), (82220, 41192), (48225, 41272), (42850, 54369), (10817, 25978), (18889, 81907), (33157, 51203), (64325, 5137), (54695, 50597), (32244, 99170), (64264, 23137), (22912, 47267), (70570, 51635), (96224, 20571), (91110, 65969), (2598, 66368), (80926, 52434), (1538, 77141), (10149, 21484), (4981, 58501), (5154, 55414), (66791, 35619), (9905, 90667), (83595, 75525), (31989, 37492), (41740, 48920), (33876, 10878), (48654, 50120), (85890, 46200), (67100, 30387), (76475, 38916), (1654, 1657), (93842, 71472), (14560, 97747), (24866, 74187), (11477, 2492), (94590, 95965), (29363, 58015), (58783, 5304), (27337, 87412), (17695, 41885), (91837, 75700), (271, 49400), (33438, 84607), (78520, 46982), (38817, 66995), (2031, 58147), (80354, 75561), (7752, 55141), (22715, 29219), (10069, 69521), (1711, 23325), (88218, 94290), (44895, 32844), (62152, 964), (32945, 57419), (19030, 51353), (53097, 36525), (76592, 2389), (53949, 27282), (10569, 90204), (94318, 48527), (13855, 27099), (41107, 11962), (39143, 81444), (66004, 38387), (55906, 61394), (34933, 33952), (53868, 82829), (86213, 80445), (40351, 56305), (97656, 90533), (48021, 7262), (49406, 38294), (57170, 7675), (7040, 8039), (65129, 76668), (7554, 74457), (65930, 2421), (12739, 37328), (17174, 68325), (63207, 21315), (70066, 44503), (19112, 8081), (80329, 65152), (76480, 39600), (75262, 45432), (24382, 67054), (75360, 90438), (21786, 19182), (2050, 82428), (34831, 92670), (69746, 44939), (6130, 58630), (62478, 14374), (35730, 71761), (20641, 66145), (86551, 64076), (37559, 79379), (8215, 82203), (33701, 9187), (22679, 38196), (17476, 25994), (4391, 62658), (87726, 32997), (22552, 65818), (18115, 42587), (64374, 97689), (64896, 49572), (98479, 75547), (881, 32287), (25305, 16000), (295, 67454), (67642, 87363), (50945, 31544), (95966, 9972), (44139, 28252), (83682, 88471), (34615, 665), (13190, 85527), (32195, 81130), (22310, 12209), (27910, 60272), (40133, 5905), (29352, 36686), (76443, 64096), (79522, 79616), (63465, 96631), (95604, 90216), (88548, 12411), (63732, 50843), (1478, 11113), (41472, 43329), (35207, 78855), (95093, 22581), (67812, 1348), (96472, 75947), (18984, 63538), (84782, 72178), (53524, 46634), (95525, 15476), (82732, 89272), (59603, 2032), (74431, 78243), (47712, 51835), (82671, 67076), (63302, 96469), (77655, 95524), (11099, 31832), (77967, 32724), (85070, 36190), (93260, 32825), (51348, 34162), (40912, 26841), (27227, 21822), (15979, 96933), (76990, 90541), (36121, 90437), (85571, 9605), (85987, 46881), (86738, 82375), (80188, 48238), (64608, 41916), (79866, 9442), (28100, 95607), (10136, 19628), (6078, 59019), (8971, 11986), (75703, 209), (81982, 98701), (30455, 66367), (92825, 84403), (73144, 4923), (62835, 75533), (90925, 56381), (83853, 86786), (66906, 28576), (69683, 74216), (2171, 22531), (6591, 52559), (59362, 88732), (49865, 38016), (82881, 9151), (71356, 27365), (91463, 45944), (71040, 12264), (62116, 27681), (84941, 82284), (57515, 60713), (45227, 74196), (77502, 17765), (54055, 29484), (33708, 46220), (37344, 75102), (80480, 37942), (95782, 36781), (14092, 82457), (25284, 82297), (85150, 50659), (90037, 99122), (49584, 48321), (8028, 65210), (8162, 32091), (1609, 22211), (38548, 1023), (35864, 43021), (54196, 30754), (88535, 90185), (5518, 97226), (79272, 57758), (61381, 86419), (38344, 86877), (72629, 24694), (12563, 11747), (81456, 44023), (55536, 65195), (67741, 12182), (64250, 29636), (45252, 9834), (16576, 40098), (65924, 11123), (14090, 83443), (95180, 92697), (92452, 7957), (21244, 29717), (50253, 62030), (21401, 70735), (86803, 89625), (17066, 52757), (74570, 89660), (11647, 74336), (22463, 68191), (41693, 67542), (12994, 21955), (3180, 27966), (37236, 51099), (68568, 62778), (2671, 68542), (42771, 4268), (7578, 12919), (33838, 61136), (87904, 48076), (3781, 57164), (83289, 14414), (29626, 91409), (92536, 13946), (88125, 34961), (71541, 41249), (21045, 85183), (21628, 62294), (17768, 22390), (5892, 10231), (85902, 84414), (44078, 87944), (64981, 31018), (85690, 11077), (4711, 9309), (39077, 26339), (85313, 99603), (20267, 31090), (92684, 11726), (17887, 17415), (23064, 83141), (40925, 87246), (17575, 71065), (29396, 86192), (10859, 2881), (50457, 42309), (86972, 41824), (80954, 76970), (11664, 94322), (4286, 93562), (9093, 32589), (81261, 96519), (67886, 3086), (74132, 43197), (94549, 26192), (55498, 75374), (93813, 32911), (6727, 51629), (93543, 55373), (48682, 83316), (32468, 77339), (60028, 4896), (34177, 67214), (35429, 16686), (72814, 39944), (82791, 97364), (21249, 36007), (85536, 84815), (32784, 34349), (2391, 45661), (62595, 30434), (56513, 65051), (22536, 99421), (80603, 62636), (23609, 19082), (33262, 63428), (45307, 96049), (75079, 62835), (52361, 19400), (43845, 89033), (54257, 33298), (31133, 35836), (56725, 10158), (73650, 10456), (77548, 28249), (57998, 79546), (97895, 93196), (472, 18004), (67178, 2684), (34612, 36352), (84970, 98816), (32474, 5895), (60388, 75957), (81643, 26457), (30070, 55744), (14725, 58396), (66451, 69932), (59264, 59569), (98045, 63704), (11925, 87468), (24701, 6280), (47797, 84240), (998, 13575), (37865, 72036), (71339, 10339), (28248, 7481), (33945, 97829), (85194, 8904), (39669, 73192), (24326, 65934), (14438, 72729), (76676, 35176), (97604, 95931), (36032, 61056), (65788, 20291), (73067, 60718), (75742, 27362), (45393, 74792), (49586, 13048), (57961, 68534), (78171, 47271), (76064, 41030), (88738, 10155), (87757, 63134), (95783, 76438), (77935, 85529), (79345, 69555), (94483, 58668), (9432, 40485), (43585, 376), (61737, 97784), (1593, 37078), (11129, 23793), (94341, 64063), (41896, 70446), (77473, 33532), (66748, 65844), (17802, 27920), (22891, 12901), (40435, 43348), (27956, 16989), (29550, 63725), (49453, 61216), (25570, 97985), (39599, 83449), (96373, 12817), (48357, 40473), (99949, 33182), (85434, 45421), (28033, 46182), (80932, 26691), (99522, 29180), (70868, 31903), (63217, 51504), (70267, 29484), (36096, 28371), (45710, 2823), (85697, 3912), (25410, 70358), (73558, 82834), (28617, 80585), (81874, 57512), (76797, 45045), (22091, 84124), (68372, 42189), (66320, 83215), (23013, 22530), (33735, 97800), (54318, 99061), (18025, 39513), (69059, 39780), (58425, 39790), (10222, 96791), (17061, 3138), (92416, 5773), (20023, 31736), (47846, 74511), (52598, 26398), (3738, 44490), (53346, 79223), (31991, 57557), (34592, 98693), (86926, 20881), (14108, 87654), (4889, 34982), (77283, 73201), (13668, 22633), (50763, 95152), (97292, 54613), (22782, 30081), (21536, 46551), (45965, 4172), (75537, 79254), (41267, 86819), (57723, 75200), (60404, 27252), (80305, 57892), (50103, 24015), (69409, 13132), (30717, 56384), (54568, 79877), (8584, 8359), (41416, 64392), (32898, 92944), (5262, 35907), (88652, 77951), (16619, 2797), (42494, 65581), (21009, 18555), (29084, 28068), (29464, 35568), (7288, 52066), (32378, 48725), (95300, 62168), (52295, 97110), (36862, 87786), (12224, 86684), (85883, 34167), (45369, 76651), (95460, 42855), (34354, 87238), (7214, 96375), (58720, 87947), (55009, 10506), (98506, 6081), (46068, 81113), (99065, 58657), (81820, 28081), (90611, 57235), (7661, 72835), (93195, 90418), (34452, 20146), (39857, 78712), (47185, 52074), (9879, 51807), (92457, 92377), (36082, 17723), (21905, 43398), (35902, 54380), (21791, 6804), (50774, 35200), (11540, 83934), (54574, 57602), (99878, 35568), (28171, 68210), (58132, 17222), (91755, 80585), (18944, 65785), (78535, 70725), (42599, 65291), (62274, 97160), (76487, 23798), (34741, 36988), (7286, 61119), (36824, 29665), (82478, 60722), (90995, 28025), (71438, 40230), (83152, 45104), (1812, 91619)) - + SELECT (rand(), rand()) IN tuple(tuple(17258, 93148), tuple(4508, 52749), tuple(68660, 70017), tuple(77797, 23528), tuple(1136, 37393), tuple(53237, 15379), tuple(68370, 73211), tuple(15782, 54962), tuple(59432, 45415), tuple(68396, 920), tuple(96154, 21016), tuple(12700, 26887), tuple(88016, 43191), tuple(68153, 51575), tuple(91315, 40005), tuple(18070, 73178), tuple(86, 631), tuple(77717, 20324), tuple(3227, 76188), tuple(74960, 43147), tuple(77538, 19628), tuple(82292, 6525), tuple(24293, 12566), tuple(85244, 96287), tuple(93982, 1329), tuple(38064, 54723), tuple(83999, 45810), tuple(71921, 53673), tuple(88638, 9669), tuple(1959, 39535), tuple(82235, 95796), tuple(27907, 90975), tuple(42383, 91015), tuple(9948, 91514), tuple(81712, 47309), tuple(400, 25808), tuple(31791, 46948), tuple(39740, 36098), tuple(25943, 84598), tuple(99598, 52939), tuple(77134, 15845), tuple(40313, 72174), tuple(85017, 94036), tuple(36595, 14303), tuple(83961, 68078), tuple(55792, 72759), tuple(73574, 43606), tuple(9853, 63560), tuple(28580, 56721), tuple(74804, 41025), tuple(32095, 55657), tuple(52881, 63416), tuple(91368, 90310), tuple(23922, 38883), tuple(30592, 10758), tuple(66448, 61183), tuple(31880, 96697), tuple(11362, 20633), tuple(75331, 2015), tuple(71129, 8785), tuple(1115, 70955), tuple(7886, 83698), tuple(18961, 84556), tuple(16677, 43028), tuple(37347, 70220), tuple(31699, 71244), tuple(10578, 96159), tuple(67600, 39041), tuple(78791, 86687), tuple(21545, 54174), tuple(68774, 37637), tuple(46132, 81768), tuple(98413, 20605), tuple(2960, 23665), tuple(31507, 35719), tuple(96209, 18368), tuple(60558, 38035), tuple(21952, 3264), tuple(11834, 86458), tuple(21651, 17650), tuple(86276, 36087), tuple(18818, 24849), tuple(61951, 3390), tuple(59637, 62545), tuple(30346, 72253), tuple(36281, 2992), tuple(78340, 49872), tuple(94326, 93723), tuple(3416, 94405), tuple(12272, 8741), tuple(22600, 22095), tuple(57636, 37106), tuple(38702, 14889), tuple(70238, 11276), tuple(17325, 60648), tuple(16492, 41271), tuple(52100, 1304), tuple(93416, 7795), tuple(57209, 71008), tuple(48010, 36078), tuple(20384, 74420), tuple(77440, 34439), tuple(69224, 45099), tuple(30374, 33884), tuple(49038, 90140), tuple(1154, 84725), tuple(64926, 86985), tuple(91746, 73472), tuple(59757, 75755), tuple(45860, 71557), tuple(45833, 36526), tuple(74618, 73598), tuple(91360, 65168), tuple(58029, 30793), tuple(56332, 14973), tuple(99943, 96877), tuple(97454, 6450), tuple(64502, 77301), tuple(73182, 31853), tuple(76809, 83964), tuple(82916, 86188), tuple(78736, 65427), tuple(36495, 7422), tuple(76196, 2804), tuple(96117, 61093), tuple(9177, 26099), tuple(52942, 63007), tuple(48578, 47876), tuple(50638, 89903), tuple(7113, 97316), tuple(35301, 12750), tuple(47807, 7254), tuple(38217, 55418), tuple(56970, 41687), tuple(20527, 62886), tuple(358, 14021), tuple(64018, 18582), tuple(91740, 21683), tuple(81967, 53589), tuple(45437, 38450), tuple(45476, 67752), tuple(76851, 72072), tuple(7304, 60091), tuple(40097, 12897), tuple(39906, 29247), tuple(84262, 58734), tuple(30857, 43791), tuple(56087, 78929), tuple(20498, 45954), tuple(48726, 500), tuple(62723, 43763), tuple(28368, 30756), tuple(74048, 52403), tuple(15045, 95926), tuple(75542, 55384), tuple(52543, 22525), tuple(56001, 6935), tuple(11431, 46745), tuple(77731, 7310), tuple(36718, 59909), tuple(32235, 91254), tuple(92417, 25917), tuple(21782, 79277), tuple(46378, 87536), tuple(35324, 26075), tuple(6310, 76915), tuple(1551, 69473), tuple(50642, 68865), tuple(55190, 72934), tuple(49780, 21873), tuple(99466, 29686), tuple(90761, 13179), tuple(72959, 57033), tuple(20020, 90200), tuple(46186, 79105), tuple(73871, 52382), tuple(59559, 38801), tuple(59916, 16082), tuple(33610, 94966), tuple(46001, 45225), tuple(86679, 26469), tuple(77245, 91929), tuple(32887, 36623), tuple(11179, 46898), tuple(87881, 68087), tuple(45438, 47991), tuple(24950, 94525), tuple(91664, 51656), tuple(43914, 47805), tuple(15736, 96156), tuple(56346, 20283), tuple(85053, 48931), tuple(17790, 26179), tuple(96195, 55728), tuple(43765, 54807), tuple(44988, 89269), tuple(55911, 99411), tuple(52446, 47397), tuple(28346, 65442), tuple(96669, 68226), tuple(66194, 26848), tuple(37276, 55864), tuple(14116, 41583), tuple(18058, 16317), tuple(93136, 85318), tuple(35616, 86252), tuple(29222, 29969), tuple(33386, 85372), tuple(71094, 44238), tuple(27733, 31838), tuple(64626, 16692), tuple(52904, 97899), tuple(97619, 12663), tuple(50165, 4688), tuple(67557, 44053), tuple(69184, 66269), tuple(73164, 89705), tuple(39822, 15169), tuple(65499, 72808), tuple(30068, 63697), tuple(30154, 64235), tuple(97016, 58716), tuple(94366, 36592), tuple(1592, 16261), tuple(87985, 52102), tuple(12554, 23652), tuple(15909, 25292), tuple(2527, 91531), tuple(92139, 36031), tuple(28986, 30032), tuple(3038, 56314), tuple(32239, 26707), tuple(15973, 34901), tuple(70246, 39680), tuple(82529, 38132), tuple(45827, 74783), tuple(53665, 64111), tuple(55218, 84170), tuple(20466, 16130), tuple(55734, 71203), tuple(31438, 96906), tuple(66338, 85858), tuple(35988, 68511), tuple(78391, 15191), tuple(80747, 59213), tuple(5357, 11546), tuple(16822, 16607), tuple(36607, 41106), tuple(74949, 30739), tuple(45726, 64887), tuple(1524, 54847), tuple(37371, 89195), tuple(28726, 27788), tuple(22600, 44777), tuple(53999, 63625), tuple(84304, 98338), tuple(49260, 76480), tuple(74564, 53907), tuple(89867, 97096), tuple(60157, 61299), tuple(17165, 10146), tuple(56334, 36268), tuple(62114, 49222), tuple(22715, 23620), tuple(42830, 11539), tuple(41091, 69151), tuple(75471, 68364), tuple(18681, 43249), tuple(42738, 63219), tuple(35474, 98454), tuple(76815, 46024), tuple(66310, 36521), tuple(86095, 77013), tuple(63693, 77319), tuple(80731, 63031), tuple(95478, 92387), tuple(23787, 63724), tuple(46299, 68994), tuple(4800, 2460), tuple(9663, 80639), tuple(77231, 85814), tuple(81615, 11311), tuple(35638, 27340), tuple(13598, 14322), tuple(30657, 17238), tuple(90957, 96846), tuple(69962, 52140), tuple(41681, 65962), tuple(96836, 58177), tuple(36190, 11623), tuple(4231, 40500), tuple(43049, 41949), tuple(71177, 98492), tuple(30193, 39750), tuple(19744, 33204), tuple(63358, 30210), tuple(45638, 58918), tuple(43641, 38741), tuple(35598, 40932), tuple(33238, 36236), tuple(50835, 20968), tuple(25099, 34071), tuple(84986, 88456), tuple(35333, 1529), tuple(79771, 23985), tuple(647, 61658), tuple(9424, 11743), tuple(77766, 31528), tuple(77811, 86973), tuple(76403, 74377), tuple(55568, 79251), tuple(68858, 20762), tuple(68520, 66773), tuple(93598, 89823), tuple(8080, 82539), tuple(87760, 52247), tuple(25191, 16905), tuple(17837, 8339), tuple(85177, 59050), tuple(51680, 77374), tuple(3287, 43018), tuple(43479, 62141), tuple(34909, 46322), tuple(11869, 5885), tuple(96193, 58417), tuple(101, 47460), tuple(34937, 88582), tuple(83216, 88388), tuple(28571, 15292), tuple(66683, 62613), tuple(34478, 8924), tuple(2680, 89973), tuple(62438, 44460), tuple(11724, 4791), tuple(5383, 72888), tuple(88206, 67586), tuple(8124, 21690), tuple(28779, 75789), tuple(66791, 4757), tuple(6176, 47760), tuple(6403, 78084), tuple(78122, 35446), tuple(99494, 73608), tuple(39691, 89098), tuple(59182, 19484), tuple(25389, 98963), tuple(96487, 3692), tuple(76222, 67381), tuple(21199, 50358), tuple(95998, 58137), tuple(28777, 43913), tuple(14176, 60117), tuple(52257, 81703), tuple(14604, 13438), tuple(71301, 14401), tuple(19758, 66914), tuple(15506, 29873), tuple(87205, 29449), tuple(93295, 15930), tuple(63651, 11287), tuple(19785, 15966), tuple(30795, 75112), tuple(69462, 37655), tuple(18793, 85764), tuple(36240, 31236), tuple(98153, 73724), tuple(72491, 4223), tuple(66930, 35048), tuple(25686, 13269), tuple(13940, 13259), tuple(69163, 11235), tuple(1183, 86961), tuple(54323, 67315), tuple(85044, 60872), tuple(48875, 3683), tuple(43052, 92861), tuple(87574, 32969), tuple(92552, 80564), tuple(94832, 47682), tuple(72011, 80994), tuple(60182, 917), tuple(97788, 34169), tuple(66432, 47940), tuple(87468, 80954), tuple(35385, 68758), tuple(50555, 63710), tuple(55311, 44337), tuple(87065, 26514), tuple(84581, 98736), tuple(23212, 56499), tuple(75120, 72447), tuple(56087, 38285), tuple(58171, 45629), tuple(28401, 44319), tuple(70432, 27883), tuple(18891, 14646), tuple(26206, 49924), tuple(79957, 44914), tuple(56064, 27529), tuple(99090, 29197), tuple(49435, 340), tuple(53525, 65601), tuple(76998, 88349), tuple(50416, 70860), tuple(42506, 75290), tuple(34024, 13295), tuple(86663, 46523), tuple(88814, 231), tuple(57809, 21), tuple(84914, 84771), tuple(43042, 66892), tuple(17288, 33908), tuple(4934, 63195), tuple(50590, 1516), tuple(97843, 80208), tuple(20091, 86717), tuple(71566, 15929), tuple(19531, 23634), tuple(41646, 45549), tuple(89226, 82902), tuple(96683, 63386), tuple(31072, 53788), tuple(51135, 41099), tuple(78912, 65609), tuple(36094, 23603), tuple(88403, 51455), tuple(73795, 47066), tuple(26448, 82852), tuple(22829, 2894), tuple(30041, 92548), tuple(27733, 20608), tuple(70180, 19892), tuple(51650, 63440), tuple(76328, 13666), tuple(40514, 6677), tuple(2786, 51059), tuple(40809, 16499), tuple(10857, 82541), tuple(78221, 61067), tuple(17982, 51969), tuple(85369, 66965), tuple(47153, 47149), tuple(43965, 75796), tuple(82725, 60767), tuple(42407, 97249), tuple(51475, 81224), tuple(60957, 89414), tuple(33065, 21663), tuple(36601, 5290), tuple(95842, 67301), tuple(64630, 60398), tuple(55212, 35638), tuple(41750, 44235), tuple(75260, 82400), tuple(91291, 25843), tuple(6477, 8311), tuple(14919, 52306), tuple(66220, 33180), tuple(45736, 2313), tuple(37450, 64444), tuple(98614, 61344), tuple(75007, 50946), tuple(56701, 28117), tuple(66632, 5174), tuple(92323, 76613), tuple(6796, 73695), tuple(33696, 76280), tuple(86876, 5614), tuple(50863, 67993), tuple(36068, 17049), tuple(91912, 34271), tuple(70706, 1904), tuple(97798, 41117), tuple(68154, 72483), tuple(83862, 25578), tuple(61643, 17204), tuple(69974, 64232), tuple(77926, 19637), tuple(64901, 88988), tuple(71424, 91703), tuple(91655, 17147), tuple(46872, 56530), tuple(44189, 98087), tuple(95939, 54420), tuple(72651, 68785), tuple(67624, 84875), tuple(92587, 87663), tuple(65275, 81256), tuple(53798, 2506), tuple(14702, 3638), tuple(71291, 50452), tuple(14909, 13903), tuple(66965, 26606), tuple(14127, 60345), tuple(35306, 1738), tuple(77234, 10468), tuple(53521, 41218), tuple(80681, 82583), tuple(44227, 26521), tuple(32263, 21482), tuple(82270, 56963), tuple(50580, 80567), tuple(11593, 22346), tuple(20074, 26867), tuple(73126, 28667), tuple(62996, 24317), tuple(20295, 57163), tuple(1506, 57668), tuple(69567, 45236), tuple(43366, 26001), tuple(88052, 40181), tuple(1599, 89349), tuple(36789, 1579), tuple(39895, 46673), tuple(30381, 3206), tuple(31723, 5625), tuple(19252, 31317), tuple(16932, 77149), tuple(48794, 34409), tuple(55986, 30328), tuple(47551, 75088), tuple(57363, 78365), tuple(95221, 63385), tuple(26449, 5733), tuple(96588, 53077), tuple(52980, 41140), tuple(8187, 85947), tuple(36723, 26520), tuple(23579, 38909), tuple(33350, 19275), tuple(63930, 19357), tuple(43536, 59941), tuple(31117, 77322), tuple(44638, 94812), tuple(44730, 99097), tuple(95108, 48170), tuple(57813, 49503), tuple(79959, 89436), tuple(86980, 62031), tuple(8275, 44009), tuple(36666, 94645), tuple(22064, 38882), tuple(40471, 16939), tuple(31156, 11337), tuple(13101, 96977), tuple(17906, 26835), tuple(89861, 51405), tuple(73369, 67946), tuple(99141, 58572), tuple(27131, 98703), tuple(15900, 43412), tuple(51768, 93125), tuple(78579, 46689), tuple(23029, 13895), tuple(60870, 55830), tuple(22553, 8236), tuple(76449, 96207), tuple(83766, 51024), tuple(27630, 50614), tuple(53484, 90104), tuple(77626, 21944), tuple(46755, 41583), tuple(53616, 34240), tuple(94159, 44415), tuple(13914, 90059), tuple(44387, 89012), tuple(27499, 64579), tuple(83415, 30809), tuple(77558, 82619), tuple(88880, 9814), tuple(8466, 4424), tuple(43598, 91921), tuple(24695, 3349), tuple(46295, 65208), tuple(51256, 82461), tuple(49126, 93012), tuple(16186, 96585), tuple(43284, 22655), tuple(93130, 90393), tuple(77495, 34372), tuple(85509, 65856), tuple(86662, 61906), tuple(50988, 44393), tuple(29828, 17737), tuple(91651, 35308), tuple(29796, 49716), tuple(14019, 87751), tuple(29688, 71207), tuple(82845, 19100), tuple(11989, 50132), tuple(21158, 99905), tuple(54732, 42547), tuple(32314, 12851), tuple(46405, 43794), tuple(87849, 45643), tuple(53524, 21212), tuple(61925, 75491), tuple(12498, 21937), tuple(30185, 69475), tuple(48421, 52487), tuple(15112, 90935), tuple(33187, 17801), tuple(61704, 25514), tuple(17889, 23917), tuple(18758, 57197), tuple(7693, 47232), tuple(47905, 24618), tuple(11494, 78950), tuple(95662, 54561), tuple(8075, 33909), tuple(90427, 46065), tuple(73962, 19821), tuple(50691, 79400), tuple(58218, 4881), tuple(94106, 2509), tuple(60633, 55169), tuple(49600, 83054), tuple(23339, 13270), tuple(70262, 58946), tuple(48417, 97266), tuple(27629, 46905), tuple(74465, 75514), tuple(41687, 2564), tuple(12814, 19492), tuple(78899, 30168), tuple(17745, 35206), tuple(37972, 35296), tuple(22288, 80001), tuple(68026, 36558), tuple(40187, 12234), tuple(92380, 22866), tuple(56488, 64402), tuple(41404, 62562), tuple(47802, 45287), tuple(83302, 85215), tuple(58999, 85776), tuple(35158, 16804), tuple(13416, 94146), tuple(62953, 28243), tuple(83290, 19103), tuple(4564, 21789), tuple(64468, 20927), tuple(25582, 47206), tuple(57810, 18693), tuple(28938, 97986), tuple(61704, 14838), tuple(19214, 3232), tuple(12911, 25438), tuple(85802, 28837), tuple(56506, 89458), tuple(66392, 47773), tuple(68190, 43841), tuple(43044, 52214), tuple(57886, 32830), tuple(15943, 59771), tuple(37081, 89294), tuple(4032, 32960), tuple(46931, 85790), tuple(69656, 72737), tuple(28217, 39872), tuple(86170, 42776), tuple(55116, 51495), tuple(90485, 45274), tuple(60773, 36788), tuple(2193, 2636), tuple(70222, 62086), tuple(75720, 70712), tuple(17549, 51460), tuple(23609, 31515), tuple(70254, 39825), tuple(63762, 11061), tuple(13107, 15394), tuple(45916, 72130), tuple(91558, 86662), tuple(99524, 69106), tuple(93073, 29881), tuple(31724, 3007), tuple(69051, 59452), tuple(59701, 86760), tuple(4967, 82028), tuple(57404, 48226), tuple(71829, 79910), tuple(23714, 62439), tuple(73881, 67618), tuple(63269, 40085), tuple(6164, 23415), tuple(48156, 93907), tuple(18627, 16570), tuple(6676, 22991), tuple(36916, 41488), tuple(99079, 13264), tuple(32533, 99243), tuple(55505, 63339), tuple(89564, 3290), tuple(24886, 34916), tuple(91310, 9343), tuple(49779, 12740), tuple(26320, 3406), tuple(57661, 5702), tuple(10765, 57881), tuple(5518, 47638), tuple(93148, 27438), tuple(73451, 24477), tuple(84075, 96822), tuple(58883, 58883), tuple(96812, 82388), tuple(30659, 59654), tuple(24498, 95808), tuple(25591, 21834), tuple(13090, 87704), tuple(76495, 17249), tuple(75975, 84318), tuple(55459, 70426), tuple(84256, 88604), tuple(79438, 43104), tuple(45331, 7495), tuple(63619, 11123), tuple(24772, 2601), tuple(63343, 14138), tuple(39957, 98339), tuple(55595, 17823), tuple(97676, 53933), tuple(91867, 25023), tuple(64677, 67859), tuple(43737, 34315), tuple(24800, 53968), tuple(93157, 17507), tuple(24264, 35273), tuple(33889, 507), tuple(10207, 40542), tuple(40213, 57800), tuple(38321, 74160), tuple(42391, 7651), tuple(80267, 94736), tuple(52473, 79634), tuple(17075, 2531), tuple(8595, 75890), tuple(31496, 50367), tuple(16069, 79896), tuple(70067, 200), tuple(23420, 49517), tuple(1628, 45646), tuple(8916, 36794), tuple(72294, 88976), tuple(40603, 86008), tuple(91871, 71098), tuple(5447, 70998), tuple(24152, 17561), tuple(65046, 34951), tuple(56950, 9292), tuple(19244, 31385), tuple(74693, 31813), tuple(97343, 21572), tuple(38834, 135), tuple(79717, 62486), tuple(38, 10308), tuple(58035, 71344), tuple(85802, 81079), tuple(5943, 156), tuple(38735, 38867), tuple(3803, 99366), tuple(15853, 19408), tuple(62988, 62008), tuple(8316, 44684), tuple(17035, 71012), tuple(48584, 2117), tuple(75425, 37336), tuple(2405, 50420), tuple(43653, 28836), tuple(12394, 69430), tuple(54522, 4954), tuple(33359, 148), tuple(41018, 82851), tuple(79995, 55417), tuple(65008, 32342), tuple(36547, 88185), tuple(8131, 7054), tuple(38980, 20146), tuple(27976, 63039), tuple(53119, 67009), tuple(40043, 98393), tuple(29333, 51980), tuple(85818, 98405), tuple(77956, 20099), tuple(99747, 16916), tuple(11597, 50181), tuple(40961, 8262), tuple(75103, 13912), tuple(62339, 69155), tuple(3869, 85481), tuple(7053, 30956), tuple(33563, 53272), tuple(96178, 81751), tuple(99365, 88728), tuple(34447, 11164), tuple(62856, 30939), tuple(92486, 3357), tuple(56605, 35330), tuple(42180, 15137), tuple(83946, 62984), tuple(61869, 55711), tuple(52880, 49871), tuple(44588, 27387), tuple(16332, 24496), tuple(1781, 13508), tuple(56674, 95773), tuple(21328, 19628), tuple(96455, 24155), tuple(14302, 74435), tuple(54053, 24590), tuple(86642, 22177), tuple(24089, 16186), tuple(70281, 4601), tuple(18552, 70708), tuple(95442, 5895), tuple(96714, 6293), tuple(43803, 45857), tuple(93257, 18497), tuple(90032, 85086), tuple(40566, 87233), tuple(32674, 73822), tuple(95599, 49334), tuple(62745, 51898), tuple(8245, 93882), tuple(14093, 40977), tuple(47215, 53001), tuple(59737, 68452), tuple(90937, 25354), tuple(43805, 82571), tuple(81953, 68572), tuple(37298, 96262), tuple(94899, 65066), tuple(34772, 80762), tuple(55469, 1186), tuple(8734, 91665), tuple(18622, 51150), tuple(85200, 39575), tuple(65381, 15979), tuple(89734, 89656), tuple(64712, 53691), tuple(87187, 58256), tuple(8476, 89694), tuple(49935, 35239), tuple(63730, 34982), tuple(27687, 91571), tuple(87543, 15350), tuple(85208, 18781), tuple(14783, 2574), tuple(44699, 666), tuple(56440, 87617), tuple(32732, 49301), tuple(76725, 3895), tuple(10419, 90580), tuple(34725, 69476), tuple(14831, 81588), tuple(93924, 38057), tuple(38528, 99060), tuple(57136, 44206), tuple(74685, 99559), tuple(43083, 87511), tuple(43105, 35474), tuple(35582, 17560), tuple(5578, 98727), tuple(78947, 53865), tuple(32013, 95029), tuple(61552, 42674), tuple(52191, 49975), tuple(71566, 16403), tuple(78534, 16350), tuple(18520, 80501), tuple(29114, 46547), tuple(11488, 5069), tuple(89591, 82384), tuple(13741, 42318), tuple(74385, 58849), tuple(49739, 63421), tuple(83821, 6676), tuple(51997, 93321), tuple(36677, 81768), tuple(37915, 73495), tuple(47175, 6086), tuple(39989, 83110), tuple(6489, 48112), tuple(88822, 20370), tuple(12846, 13952), tuple(28930, 20879), tuple(25139, 84552), tuple(76434, 2665), tuple(55145, 31523), tuple(21177, 18630), tuple(81077, 96275), tuple(61006, 30845), tuple(77722, 62651), tuple(61181, 72545), tuple(93838, 84287), tuple(59300, 19014), tuple(75076, 97980), tuple(76979, 1473), tuple(48409, 13097), tuple(51718, 5325), tuple(36522, 72119), tuple(60917, 18995), tuple(61469, 42853), tuple(34387, 37322), tuple(38684, 28120), tuple(64136, 8559), tuple(15368, 99424), tuple(97824, 7864), tuple(33833, 72029), tuple(7024, 9961), tuple(49400, 66220), tuple(63025, 97179), tuple(6135, 98878), tuple(19873, 8438), tuple(3963, 35670), tuple(65186, 89423), tuple(26653, 65943), tuple(83132, 67000), tuple(82578, 35007), tuple(42680, 60479), tuple(71102, 98589), tuple(74842, 94010), tuple(22931, 33725), tuple(46537, 42629), tuple(75793, 48115), tuple(21630, 92454), tuple(97993, 81332), tuple(25747, 31814), tuple(91231, 65953), tuple(91981, 12219), tuple(64719, 16254), tuple(60914, 8334), tuple(15887, 96432), tuple(42110, 28837), tuple(7295, 83147), tuple(50334, 7053), tuple(3949, 33594), tuple(1524, 98230), tuple(17265, 98024), tuple(75969, 36232), tuple(89538, 5212), tuple(13444, 55946), tuple(69823, 81848), tuple(32578, 74024), tuple(52018, 98290), tuple(59118, 40186), tuple(61002, 16977), tuple(69537, 44780), tuple(92, 13937), tuple(33715, 42663), tuple(46347, 8312), tuple(86196, 59301), tuple(17128, 85014), tuple(26429, 57682), tuple(45888, 99588), tuple(22750, 96110), tuple(46809, 49251), tuple(24521, 40071), tuple(287, 22115), tuple(11741, 36315), tuple(22742, 17581), tuple(35808, 3110), tuple(98904, 30407), tuple(4584, 13383), tuple(28585, 69669), tuple(94823, 29715), tuple(9551, 36389), tuple(77997, 45746), tuple(49894, 55722), tuple(23415, 69459), tuple(58246, 85144), tuple(74136, 18102), tuple(97366, 85724), tuple(34271, 51601), tuple(47535, 70883), tuple(59443, 90103), tuple(45213, 45811), tuple(62741, 86898), tuple(17324, 50034), tuple(62080, 25193), tuple(89524, 4421), tuple(13476, 51456), tuple(69198, 56718), tuple(58024, 22969), tuple(65210, 67941), tuple(32561, 44881), tuple(62295, 67448), tuple(66135, 95453), tuple(9417, 20443), tuple(82486, 23745), tuple(19185, 99041), tuple(40662, 91714), tuple(3423, 58624), tuple(4512, 74502), tuple(67772, 98023), tuple(69575, 75779), tuple(69107, 62805), tuple(517, 33801), tuple(47406, 7581), tuple(81108, 10546), tuple(12976, 47001), tuple(16742, 83811), tuple(44593, 82124), tuple(52731, 34642), tuple(81725, 20555), tuple(94126, 91919), tuple(24800, 59302), tuple(97253, 39249), tuple(71692, 10769), tuple(88721, 56321), tuple(7019, 69771), tuple(31464, 61774), tuple(29597, 19263), tuple(65557, 31875), tuple(28653, 69636), tuple(58074, 76848), tuple(15906, 80620), tuple(18259, 40193), tuple(99991, 4769), tuple(98935, 99269), tuple(12123, 60124), tuple(20787, 47346), tuple(13526, 33592), tuple(95370, 40350), tuple(17479, 42884), tuple(58368, 83218), tuple(63290, 74406), tuple(97030, 35102), tuple(45298, 27660), tuple(64593, 21262), tuple(76268, 82641), tuple(1107, 44044), tuple(21427, 79959), tuple(85180, 62412), tuple(7359, 1318), tuple(83618, 9762), tuple(1425, 55804), tuple(32874, 97943), tuple(68191, 38742), tuple(41715, 17902), tuple(3771, 15032), tuple(7848, 74950), tuple(33881, 40904), tuple(75295, 26151), tuple(75775, 13760), tuple(90262, 89822), tuple(88169, 18679), tuple(57506, 32356), tuple(94983, 44281), tuple(37385, 37432), tuple(18248, 48162), tuple(45573, 66278), tuple(25277, 72788), tuple(26977, 36778), tuple(26254, 61758), tuple(12860, 48026), tuple(96819, 3339), tuple(13134, 1173), tuple(26822, 53374), tuple(15989, 29698), tuple(11258, 54515), tuple(37866, 34928), tuple(22996, 26577), tuple(39952, 42732), tuple(6754, 70595), tuple(86245, 44669), tuple(47044, 34170), tuple(6789, 45220), tuple(31706, 2090), tuple(42582, 40023), tuple(35147, 46591), tuple(88210, 11307), tuple(53644, 7680), tuple(11280, 91075), tuple(42961, 65122), tuple(40066, 52185), tuple(20050, 6154), tuple(98440, 20393), tuple(88992, 75432), tuple(32386, 66731), tuple(36952, 34149), tuple(18453, 32715), tuple(84413, 10378), tuple(59440, 2374), tuple(45354, 85009), tuple(50382, 66510), tuple(64428, 95401), tuple(9336, 41760), tuple(26317, 91416), tuple(81941, 99504), tuple(26600, 53522), tuple(81069, 40236), tuple(51126, 27911), tuple(97144, 14243), tuple(62738, 50287), tuple(37372, 28962), tuple(12053, 9090), tuple(69492, 95524), tuple(68141, 52931), tuple(17276, 16487), tuple(69227, 25949), tuple(14143, 70193), tuple(7077, 53032), tuple(65463, 74082), tuple(94997, 66496), tuple(80443, 55832), tuple(66796, 5970), tuple(15852, 95662), tuple(81559, 97272), tuple(55851, 18977), tuple(91142, 48976), tuple(91143, 950), tuple(79225, 31004), tuple(61310, 20760), tuple(74541, 90842), tuple(80322, 11630), tuple(84631, 544), tuple(66785, 86591), tuple(25650, 63252), tuple(59635, 18586), tuple(2964, 6741), tuple(37091, 71148), tuple(11984, 43077), tuple(87505, 62049), tuple(61925, 92290), tuple(18808, 3937), tuple(8300, 33268), tuple(70850, 50661), tuple(86024, 73730), tuple(85161, 47116), tuple(50193, 89155), tuple(37773, 40845), tuple(9251, 41688), tuple(6940, 65399), tuple(42479, 95630), tuple(19401, 43102), tuple(48069, 36040), tuple(62760, 95013), tuple(394, 2641), tuple(32567, 29306), tuple(13870, 58835), tuple(98248, 47291), tuple(49803, 4523), tuple(40222, 12883), tuple(53576, 73105), tuple(88265, 23629), tuple(67865, 67875), tuple(33473, 27144), tuple(80219, 53893), tuple(74878, 47341), tuple(78070, 84803), tuple(30003, 5600), tuple(41103, 6145), tuple(83490, 81076), tuple(55059, 66736), tuple(45015, 10239), tuple(79555, 85819), tuple(81808, 34970), tuple(19235, 85480), tuple(91807, 52177), tuple(40887, 87009), tuple(5003, 2687), tuple(64964, 88122), tuple(765, 94893), tuple(93573, 20504), tuple(28854, 38438), tuple(94244, 93475), tuple(72996, 84801), tuple(75427, 81692), tuple(63161, 98637), tuple(18814, 61343), tuple(22863, 60110), tuple(8949, 12694), tuple(19675, 94313), tuple(43857, 74073), tuple(15737, 58218), tuple(48895, 68474), tuple(22220, 92926), tuple(69055, 50282), tuple(40532, 74934), tuple(59062, 66405), tuple(85784, 87704), tuple(58494, 88222), tuple(2260, 20401), tuple(73112, 99666), tuple(46739, 95433), tuple(21179, 85119), tuple(11545, 38801), tuple(59993, 50866), tuple(10086, 4709), tuple(70560, 29611), tuple(27095, 89017), tuple(6896, 2279), tuple(92506, 5013), tuple(48600, 90491), tuple(18782, 54638), tuple(54337, 82734), tuple(52054, 13481), tuple(38297, 56559), tuple(15998, 30591), tuple(89789, 7522), tuple(18149, 28725), tuple(3532, 28625), tuple(70934, 49617), tuple(84599, 55664), tuple(74229, 52269), tuple(55431, 11893), tuple(32807, 72543), tuple(83882, 53025), tuple(11490, 83442), tuple(14844, 88612), tuple(12526, 45953), tuple(906, 2231), tuple(68240, 95612), tuple(18818, 31535), tuple(57774, 91290), tuple(67250, 67400), tuple(77332, 23550), tuple(42332, 57775), tuple(28792, 11539), tuple(19108, 34608), tuple(12399, 38591), tuple(7329, 10740), tuple(84288, 50928), tuple(29461, 17629), tuple(63884, 88489), tuple(47479, 61085), tuple(75357, 57255), tuple(60107, 94046), tuple(32934, 66312), tuple(28615, 42600), tuple(55553, 85213), tuple(57838, 91426), tuple(9783, 11513), tuple(73677, 28821), tuple(75408, 75561), tuple(22995, 59224), tuple(74874, 54145), tuple(18513, 75901), tuple(46440, 69414), tuple(36072, 22263), tuple(60560, 73325), tuple(69967, 93358), tuple(75949, 98634), tuple(3688, 57991), tuple(43482, 94541), tuple(40922, 31011), tuple(57763, 74497), tuple(93576, 96392), tuple(83038, 80656), tuple(47757, 87045), tuple(14061, 53465), tuple(65619, 33775), tuple(11341, 6702), tuple(6249, 87358), tuple(15766, 85937), tuple(13135, 93945), tuple(24495, 95900), tuple(80359, 1739), tuple(15468, 73426), tuple(49240, 44999), tuple(82839, 90808), tuple(87438, 75613), tuple(348, 73144), tuple(99523, 85853), tuple(21557, 70210), tuple(64933, 1672), tuple(38154, 17477), tuple(97136, 67363), tuple(96491, 8038), tuple(97981, 3434), tuple(54372, 27038), tuple(88480, 86675), tuple(21028, 21083), tuple(43197, 4440), tuple(31702, 78290), tuple(66631, 24438), tuple(11482, 17922), tuple(90351, 39503), tuple(46186, 32439), tuple(73828, 6640), tuple(56916, 26029), tuple(62840, 1815), tuple(20281, 28488), tuple(18211, 30043), tuple(65211, 93012), tuple(43614, 58012), tuple(90322, 77343), tuple(64293, 94525), tuple(59489, 39760), tuple(93219, 78440), tuple(74613, 9732), tuple(38085, 19191), tuple(58029, 48186), tuple(88762, 1764), tuple(28627, 21993), tuple(49975, 41225), tuple(70486, 43480), tuple(82764, 96425), tuple(27218, 78327), tuple(17844, 73333), tuple(70463, 37629), tuple(10500, 33826), tuple(97343, 66575), tuple(82833, 51210), tuple(77353, 45073), tuple(27163, 39728), tuple(78076, 46691), tuple(80302, 39342), tuple(77142, 1319), tuple(87403, 80110), tuple(53805, 27786), tuple(50558, 74264), tuple(83146, 31358), tuple(11567, 4438), tuple(30041, 54287), tuple(91731, 18496), tuple(57591, 93894), tuple(72534, 59009), tuple(98064, 59148), tuple(69626, 66615), tuple(20951, 43949), tuple(61960, 68060), tuple(48892, 67918), tuple(61321, 56222), tuple(75424, 77260), tuple(4916, 81929), tuple(68892, 81531), tuple(28096, 28548), tuple(62016, 107), tuple(8593, 12030), tuple(66743, 36772), tuple(60174, 15106), tuple(52844, 1923), tuple(34768, 22065), tuple(88988, 62910), tuple(79214, 2998), tuple(25675, 31376), tuple(69959, 3614), tuple(43885, 31708), tuple(12206, 46548), tuple(69924, 19343), tuple(12984, 38980), tuple(58250, 69438), tuple(2580, 48684), tuple(38112, 37124), tuple(21842, 43150), tuple(59384, 21921), tuple(19908, 46678), tuple(73396, 79529), tuple(8274, 1557), tuple(36975, 65519), tuple(81069, 18712), tuple(13692, 9148), tuple(60617, 84762), tuple(75749, 66154), tuple(80375, 24553), tuple(4257, 47056), tuple(76880, 7687), tuple(40714, 43448), tuple(79112, 74791), tuple(33119, 72730), tuple(17670, 89183), tuple(51614, 3921), tuple(21247, 39857), tuple(86756, 67673), tuple(32792, 70035), tuple(5917, 7197), tuple(1762, 23130), tuple(6455, 63664), tuple(32806, 3729), tuple(60469, 20511), tuple(12522, 15149), tuple(98106, 79338), tuple(84754, 11162), tuple(52058, 17973), tuple(28789, 1521), tuple(32766, 36325), tuple(78914, 40453), tuple(70297, 71854), tuple(9313, 45190), tuple(54559, 66227), tuple(22342, 43860), tuple(44152, 84294), tuple(36913, 93173), tuple(88523, 36338), tuple(82234, 71140), tuple(8328, 22947), tuple(73250, 88125), tuple(74356, 16820), tuple(94472, 37349), tuple(23126, 87806), tuple(40315, 88729), tuple(19935, 19145), tuple(93312, 65719), tuple(8477, 33108), tuple(86660, 69525), tuple(75557, 66964), tuple(60437, 57494), tuple(94419, 42524), tuple(95372, 72274), tuple(49866, 85685), tuple(96808, 39404), tuple(62961, 72507), tuple(38634, 70815), tuple(91379, 42430), tuple(66359, 98699), tuple(24382, 4186), tuple(4003, 77760), tuple(87840, 75265), tuple(57641, 68871), tuple(9773, 15942), tuple(5664, 51289), tuple(47923, 31308), tuple(58632, 82468), tuple(14097, 71829), tuple(1838, 97710), tuple(70433, 11364), tuple(82363, 97879), tuple(25257, 20615), tuple(18249, 6758), tuple(98581, 13639), tuple(3290, 72449), tuple(74546, 79380), tuple(97254, 44448), tuple(80316, 31760), tuple(40516, 94809), tuple(14444, 88981), tuple(9693, 10259), tuple(83795, 95485), tuple(70201, 81014), tuple(66644, 16761), tuple(35529, 82718), tuple(75774, 73476), tuple(80139, 3957), tuple(34803, 80689), tuple(92085, 46499), tuple(97871, 8004), tuple(67369, 11354), tuple(43578, 81596), diff --git a/tests/performance/sum_map.xml b/tests/performance/sum_map.xml index b732c150220..ff166bbf15f 100644 --- a/tests/performance/sum_map.xml +++ b/tests/performance/sum_map.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/synthetic_hardware_benchmark.xml b/tests/performance/synthetic_hardware_benchmark.xml index ffcf30db5cb..4b94f73a21d 100644 --- a/tests/performance/synthetic_hardware_benchmark.xml +++ b/tests/performance/synthetic_hardware_benchmark.xml @@ -1,4 +1,4 @@ - + 30000000000 diff --git a/tests/performance/url_hits.xml b/tests/performance/url_hits.xml index 1813b2a72cb..a699ef6ba97 100644 --- a/tests/performance/url_hits.xml +++ b/tests/performance/url_hits.xml @@ -1,4 +1,4 @@ - + hits_100m_single hits_10m_single diff --git a/tests/performance/visit_param_extract_raw.xml b/tests/performance/visit_param_extract_raw.xml index 358dcc9cc0e..67faeb1f743 100644 --- a/tests/performance/visit_param_extract_raw.xml +++ b/tests/performance/visit_param_extract_raw.xml @@ -1,4 +1,4 @@ - + param diff --git a/tests/queries/0_stateless/01072_nullable_jit.sql b/tests/queries/0_stateless/01072_nullable_jit.sql index 45e9f110bcb..ae220fda475 100644 --- a/tests/queries/0_stateless/01072_nullable_jit.sql +++ b/tests/queries/0_stateless/01072_nullable_jit.sql @@ -1,3 +1,5 @@ +SET compile_expressions = 1; + DROP TABLE IF EXISTS foo; CREATE TABLE foo ( @@ -12,7 +14,7 @@ CREATE TABLE foo ( INSERT INTO foo VALUES (1, 0.5, 0.2, 0.3, 0.8); -SELECT divide(sum(a) + sum(b), nullIf(sum(c) + sum(d), 0)) FROM foo SETTINGS compile_expressions = 1; -SELECT divide(sum(a) + sum(b), nullIf(sum(c) + sum(d), 0)) FROM foo SETTINGS compile_expressions = 1; +SELECT divide(sum(a) + sum(b), nullIf(sum(c) + sum(d), 0)) FROM foo; +SELECT divide(sum(a) + sum(b), nullIf(sum(c) + sum(d), 0)) FROM foo; DROP TABLE foo; diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.reference b/tests/queries/0_stateless/01085_max_distributed_connections.reference index bc6ff2b709c..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01085_max_distributed_connections.reference +++ b/tests/queries/0_stateless/01085_max_distributed_connections.reference @@ -1,9 +0,0 @@ -0 -0 -0 -0 -0 -0 -0 -0 -0 diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.sh b/tests/queries/0_stateless/01085_max_distributed_connections.sh index c63c671f7fc..1c42cd1a7e0 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections.sh @@ -4,11 +4,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -opts=( - --max_distributed_connections 9 - --max_threads 1 - --query "SELECT sleepEachRow(1) FROM remote('127.{2..10}', system.one)" -) -# 5 less then 9 seconds (9 streams), but long enough to cover possible load peaks -# "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing -timeout 5s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" +# Sometimes five seconds are not enough due to system overload. +# But if it can run in less than five seconds at least sometimes - it is enough for the test. +while true +do + opts=( + --max_distributed_connections 9 + --max_threads 1 + --query "SELECT sleepEachRow(1) FROM remote('127.{2..10}', system.one)" + --format Null + ) + # 5 less then 9 seconds (9 streams), but long enough to cover possible load peaks + # "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing + + timeout 5s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break +done diff --git a/tests/queries/0_stateless/01085_max_distributed_connections_http.reference b/tests/queries/0_stateless/01085_max_distributed_connections_http.reference index aa47d0d46d4..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01085_max_distributed_connections_http.reference +++ b/tests/queries/0_stateless/01085_max_distributed_connections_http.reference @@ -1,2 +0,0 @@ -0 -0 diff --git a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh index d7eb75e717a..1aa9939407c 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh @@ -4,6 +4,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -query="SELECT sleepEachRow(1) FROM remote('127.{2,3}', system.one)" -# 1.8 less then 2 seconds, but long enough to cover possible load peaks -timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" +# Sometimes 1.8 seconds are not enough due to system overload. +# But if it can run in less than five seconds at least sometimes - it is enough for the test. + +while true +do + query="SELECT sleepEachRow(1) FROM remote('127.{2,3}', system.one) FORMAT Null" + # 1.8 less then 2 seconds, but long enough to cover possible load peaks + timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" && break +done diff --git a/tests/queries/0_stateless/01155_old_mutation_parts_to_do.reference b/tests/queries/0_stateless/01155_old_mutation_parts_to_do.reference new file mode 100644 index 00000000000..725b16fe1be --- /dev/null +++ b/tests/queries/0_stateless/01155_old_mutation_parts_to_do.reference @@ -0,0 +1,7 @@ +waiting default rmt 0000000002 UPDATE m = m * toInt8(s) WHERE n = 3 +1 4 2 +2 15 5 +3 7 fail +4 11 13 +0000000000 UPDATE m = m * toInt8(s) WHERE 1 [] 0 1 +0000000001 UPDATE m = m * toInt8(s) WHERE 1 [] 0 1 diff --git a/tests/queries/0_stateless/01155_old_mutation_parts_to_do.sh b/tests/queries/0_stateless/01155_old_mutation_parts_to_do.sh new file mode 100755 index 00000000000..0354da26f4f --- /dev/null +++ b/tests/queries/0_stateless/01155_old_mutation_parts_to_do.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./mergetree_mutations.lib +. "$CURDIR"/mergetree_mutations.lib + + +${CLICKHOUSE_CLIENT} -q "drop table if exists rmt;" + +${CLICKHOUSE_CLIENT} -q "create table rmt (n int, m int, s String) engine=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') +order by n settings max_replicated_mutations_in_queue=0;" + +${CLICKHOUSE_CLIENT} -q "insert into rmt values (1, 1, '2');" # 0_0_0_0 +${CLICKHOUSE_CLIENT} --mutations_sync=0 -q "alter table rmt update m = m*toInt8(s) where 1;" # 0000000000 +${CLICKHOUSE_CLIENT} -q "insert into rmt values (2, 3, '5');" # 0_2_2_0 +${CLICKHOUSE_CLIENT} --mutations_sync=0 -q "alter table rmt update m = m*toInt8(s) where 1;" # 0000000001 +${CLICKHOUSE_CLIENT} -q "insert into rmt values (3, 7, 'fail');" # 0_4_4_0 +${CLICKHOUSE_CLIENT} --mutations_sync=0 -q "alter table rmt update m = m*toInt8(s) where n=3;" # 0000000002, will fail to mutate 0_4_4_0 to 0_4_4_0_5 +${CLICKHOUSE_CLIENT} -q "insert into rmt values (4, 11, '13');" # 0_6_6_0 + +${CLICKHOUSE_CLIENT} -q "alter table rmt modify setting max_replicated_mutations_in_queue=1;" +sleep 5 # test does not rely on this, but it may help to reproduce a bug + +${CLICKHOUSE_CLIENT} -q "kill mutation where database=currentDatabase() and table='rmt' and mutation_id='0000000002'"; +${CLICKHOUSE_CLIENT} -q "system sync replica rmt;" + +# now check that mutations 0 and 1 are finished +wait_for_mutation "rmt" "0000000001" +${CLICKHOUSE_CLIENT} -q "select * from rmt order by n;" +${CLICKHOUSE_CLIENT} -q "select mutation_id, command, parts_to_do_names, parts_to_do, is_done from system.mutations where database=currentDatabase() and table='rmt';" + +${CLICKHOUSE_CLIENT} -q "drop table rmt;" diff --git a/tests/queries/0_stateless/01414_mutations_and_errors.reference b/tests/queries/0_stateless/01414_mutations_and_errors.reference index 166a9c6b7b8..a1edd6ab649 100644 --- a/tests/queries/0_stateless/01414_mutations_and_errors.reference +++ b/tests/queries/0_stateless/01414_mutations_and_errors.reference @@ -1,4 +1,5 @@ 42 Hello +waiting default mutation_table mutation_3.txt MODIFY COLUMN `value` UInt64 42 Hello diff --git a/tests/queries/0_stateless/01871_merge_tree_compile_expressions.sql b/tests/queries/0_stateless/01871_merge_tree_compile_expressions.sql index f8cad868187..ea1e26b1f00 100644 --- a/tests/queries/0_stateless/01871_merge_tree_compile_expressions.sql +++ b/tests/queries/0_stateless/01871_merge_tree_compile_expressions.sql @@ -6,9 +6,9 @@ SET compile_expressions=true; -- CREATE TABLE will use global profile with default min_count_to_compile_expression=3 -- so retry 3 times -CREATE TABLE data_01875_1 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number,8) AS SELECT * FROM numbers(16384); -CREATE TABLE data_01875_2 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number,8) AS SELECT * FROM numbers(16384); -CREATE TABLE data_01875_3 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number,8) AS SELECT * FROM numbers(16384); +CREATE TABLE data_01875_1 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number, 8) + 1 AS SELECT * FROM numbers(16384); +CREATE TABLE data_01875_2 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number, 8) + 1 AS SELECT * FROM numbers(16384); +CREATE TABLE data_01875_3 Engine=MergeTree ORDER BY number PARTITION BY bitShiftRight(number, 8) + 1 AS SELECT * FROM numbers(16384); SELECT number FROM data_01875_3 WHERE number = 999; diff --git a/tests/queries/0_stateless/01881_union_header_mismatch_bug.reference b/tests/queries/0_stateless/01881_union_header_mismatch_bug.reference new file mode 100644 index 00000000000..fbec53380d2 --- /dev/null +++ b/tests/queries/0_stateless/01881_union_header_mismatch_bug.reference @@ -0,0 +1,11 @@ +table 10 101 +a 0 +a 1 +a 2 +a 3 +a 4 +b 0 +b 1 +b 2 +b 3 +b 4 diff --git a/tests/queries/0_stateless/01881_union_header_mismatch_bug.sql b/tests/queries/0_stateless/01881_union_header_mismatch_bug.sql new file mode 100644 index 00000000000..9a220ffd49f --- /dev/null +++ b/tests/queries/0_stateless/01881_union_header_mismatch_bug.sql @@ -0,0 +1,30 @@ +select *　from (　select 'table' as table, toInt64(10) as rows, toInt64(101) as elements　union all　select 'another table' as table, toInt64(0) as rows, toInt64(0) as elements　)　where rows - elements <> 0; + +SELECT + label, + number +FROM +( + SELECT + 'a' AS label, + number + FROM + ( + SELECT number + FROM numbers(10) + ) + UNION ALL + SELECT + 'b' AS label, + number + FROM + ( + SELECT number + FROM numbers(10) + ) +) +WHERE number IN +( + SELECT number + FROM numbers(5) +) order by label, number;