diff --git a/.gitignore b/.gitignore index 6c0865d1959..bc598cf0d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,10 @@ # auto generated files *.logrt +dbms/src/Storages/System/StorageSystemContributors.generated.cpp + /build +/build_* /docs/build /docs/edit /docs/tools/venv/ @@ -243,3 +246,6 @@ website/presentations website/package-lock.json .DS_Store */.DS_Store + +# Ignore files for locally disabled tests +/dbms/tests/queries/**/*.disabled diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index e69de29bb2d..50a7665dc63 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -0,0 +1 @@ +* Настройка `enable_optimize_predicate_expression` выключена по-умолчанию. diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index 9c033094fab..2569c655c9b 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,117 +1,129 @@ +## ClickHouse release 18.14.11, 2018-10-29 + +### Исправления ошибок: + +* Исправлена ошибка `Block structure mismatch in UNION stream: different number of columns` в запросах с LIMIT. [#2156](https://github.com/yandex/ClickHouse/issues/2156) +* Исправлены ошибки при слиянии данных в таблицах, содержащих массивы внутри Nested структур. [#3397](https://github.com/yandex/ClickHouse/pull/3397) +* Исправлен неправильный результат запросов при выключенной настройке `merge_tree_uniform_read_distribution` (включена по умолчанию). [#3429](https://github.com/yandex/ClickHouse/pull/3429) +* Исправлена ошибка при вставке в Distributed таблицу в формате Native. [#3411](https://github.com/yandex/ClickHouse/issues/3411) + +## ClickHouse release 18.14.10, 2018-10-23 + +* Настройка `compile_expressions` (JIT компиляция выражений) выключена по умолчанию. [#3410](https://github.com/yandex/ClickHouse/pull/3410) +* Настройка `enable_optimize_predicate_expression` выключена по умолчанию. + ## ClickHouse release 18.14.9, 2018-10-16 ### Новые возможности: -* Модификатор `WITH CUBE` для `GROUP BY` (также доступен синтаксис: `GROUP BY CUBE(...)`). -* Добавлена функция `formatDateTime`. [Alexandr Krasheninnikov]() -* Добавлен движок таблиц `JDBC` и табличная функция `jdbc` (для работы требуется установка clickhouse-jdbc-bridge). [Alexandr Krasheninnikov]() -* Добавлены функции для работы с ISO номером недели: `toISOWeek`, `toISOYear`, `toStartOfISOYear`. -* Добавлена функция `toDayOfYear`. -* Добавлена возможность использования столбцов типа `Nullable` для таблиц типа `MySQL`, `ODBC`. -* Возможность чтения вложенных структур данных как вложенных объектов в формате `JSONEachRow`. Добавлена настройка `input_format_import_nested_json`. [Veloman Yunkan](). -* Возможность параллельной обработки многих `MATERIALIZED VIEW` при вставке данных. Настройка `parallel_view_processing`. [Marek Vavruša](). -* Добавлен запрос `SYSTEM FLUSH LOGS` (форсированный сброс логов в системные таблицы, такие как например, `query_log`). -* Возможность использования предопределённых макросов `database` и `table` в объявлении `Replicated` таблиц. -* Добавлена возможность чтения значения типа `Decimal` в инженерной нотации (с указанием десятичной экспоненты). +* Модификатор `WITH CUBE` для `GROUP BY` (также доступен синтаксис: `GROUP BY CUBE(...)`). [#3172](https://github.com/yandex/ClickHouse/pull/3172) +* Добавлена функция `formatDateTime`. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/2770) +* Добавлен движок таблиц `JDBC` и табличная функция `jdbc` (для работы требуется установка clickhouse-jdbc-bridge). [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) +* Добавлены функции для работы с ISO номером недели: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, а также `toDayOfYear`. [#3146](https://github.com/yandex/ClickHouse/pull/3146) +* Добавлена возможность использования столбцов типа `Nullable` для таблиц типа `MySQL`, `ODBC`. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Возможность чтения вложенных структур данных как вложенных объектов в формате `JSONEachRow`. Добавлена настройка `input_format_import_nested_json`. [Veloman Yunkan](https://github.com/yandex/ClickHouse/pull/3144) +* Возможность параллельной обработки многих `MATERIALIZED VIEW` при вставке данных. Настройка `parallel_view_processing`. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3208) +* Добавлен запрос `SYSTEM FLUSH LOGS` (форсированный сброс логов в системные таблицы, такие как например, `query_log`) [#3321](https://github.com/yandex/ClickHouse/pull/3321) +* Возможность использования предопределённых макросов `database` и `table` в объявлении `Replicated` таблиц. [#3251](https://github.com/yandex/ClickHouse/pull/3251) +* Добавлена возможность чтения значения типа `Decimal` в инженерной нотации (с указанием десятичной экспоненты). [#3153](https://github.com/yandex/ClickHouse/pull/3153) ### Экспериментальные возможности: -* Оптимизация GROUP BY для типов данных `LowCardinality`. -* Оптимизации вычисления выражений для типов данных `LowCardinality`. +* Оптимизация GROUP BY для типов данных `LowCardinality` [#3138](https://github.com/yandex/ClickHouse/pull/3138) +* Оптимизации вычисления выражений для типов данных `LowCardinality` [#3200](https://github.com/yandex/ClickHouse/pull/3200) ### Улучшения: -* Существенно уменьшено потребление памяти для запросов с `ORDER BY` и `LIMIT`. Настройка `max_bytes_before_remerge_sort`. -* При отсутствии указания типа `JOIN` (`LEFT`, `INNER`, ...), подразумевается `INNER JOIN`. -* Корректная работа квалифицированных звёздочек в запросах с `JOIN`. [Winter Zhang]() -* Движок таблиц `ODBC` корректно выбирает способ квотирования идентификаторов в SQL диалекте удалённой СУБД. [Alexandr Krasheninnikov]() +* Существенно уменьшено потребление памяти для запросов с `ORDER BY` и `LIMIT`. Настройка `max_bytes_before_remerge_sort`. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* При отсутствии указания типа `JOIN` (`LEFT`, `INNER`, ...), подразумевается `INNER JOIN`. [#3147](https://github.com/yandex/ClickHouse/pull/3147) +* Корректная работа квалифицированных звёздочек в запросах с `JOIN`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3202) +* Движок таблиц `ODBC` корректно выбирает способ квотирования идентификаторов в SQL диалекте удалённой СУБД. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) * Настройка `compile_expressions` (JIT компиляция выражений) включена по-умолчанию. -* Исправлено поведение при одновременном DROP DATABASE/TABLE IF EXISTS и CREATE DATABASE/TABLE IF NOT EXISTS. Ранее запрос `CREATE DATABASE ... IF NOT EXISTS` мог выдавать сообщение об ошибке вида "File ... already exists", а запросы `CREATE TABLE ... IF NOT EXISTS` и `DROP TABLE IF EXISTS` могли выдавать сообщение `Table ... is creating or attaching right now`. -* Выражения LIKE и IN с константной правой частью пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. -* Сравнения с константными выражениями в секции WHERE пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. Ранее пробрасывались только сравнения с константами. -* Корректное вычисление ширины строк в терминале для `Pretty` форматов, в том числе для строк с иероглифами. [Amos Bird](). +* Исправлено поведение при одновременном DROP DATABASE/TABLE IF EXISTS и CREATE DATABASE/TABLE IF NOT EXISTS. Ранее запрос `CREATE DATABASE ... IF NOT EXISTS` мог выдавать сообщение об ошибке вида "File ... already exists", а запросы `CREATE TABLE ... IF NOT EXISTS` и `DROP TABLE IF EXISTS` могли выдавать сообщение `Table ... is creating or attaching right now`. [#3101](https://github.com/yandex/ClickHouse/pull/3101) +* Выражения LIKE и IN с константной правой частью пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Сравнения с константными выражениями в секции WHERE пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. Ранее пробрасывались только сравнения с константами. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Корректное вычисление ширины строк в терминале для `Pretty` форматов, в том числе для строк с иероглифами. [Amos Bird](https://github.com/yandex/ClickHouse/pull/3257). * Возможность указания `ON CLUSTER` для запросов `ALTER UPDATE`. -* Увеличена производительность чтения данных в формате `JSONEachRow`. -* Добавлены синонимы функций `LENGTH`, `CHARACTER_LENGTH` для совместимости. Функция `CONCAT` стала регистронезависимой. -* Добавлен синоним `TIMESTAMP` для типа `DateTime`. +* Увеличена производительность чтения данных в формате `JSONEachRow`. [#3332](https://github.com/yandex/ClickHouse/pull/3332) +* Добавлены синонимы функций `LENGTH`, `CHARACTER_LENGTH` для совместимости. Функция `CONCAT` стала регистронезависимой. [#3306](https://github.com/yandex/ClickHouse/pull/3306) +* Добавлен синоним `TIMESTAMP` для типа `DateTime`. [#3390](https://github.com/yandex/ClickHouse/pull/3390) * В логах сервера всегда присутствует место для query_id, даже если строчка лога не относится к запросу. Это сделано для более простого парсинга текстовых логов сервера сторонними инструментами. -* Логгирование потребления памяти запросом при превышении очередной отметки целого числа гигабайт. -* Добавлен режим совместимости для случая, когда клиентская библиотека, работающая по Native протоколу, по ошибке отправляет меньшее количество столбцов, чем сервер ожидает для запроса INSERT. Такой сценарий был возможен при использовании библиотеки clickhouse-cpp. Ранее этот сценарий приводил к падению сервера. -* В `clickhouse-copier`, в задаваемом пользователем выражении WHERE, появилась возможность использовать алиас `partition_key` (для дополнительной фильтрации по партициям исходной таблицы). Это полезно, если схема партиционирования изменяется при копировании, но изменяется незначительно. -* Рабочий поток движка `Kafka` перенесён в фоновый пул потоков для того, чтобы автоматически уменьшать скорость чтения данных при большой нагрузке. [Marek Vavruša](). -* Поддержка чтения значений типа `Tuple` и `Nested` структур как `struct` в формате `Cap'n'Proto` [Marek Vavruša](). -* В список доменов верхнего уровня для функции `firstSignificantSubdomain` добавлен домен `biz` [decaseal](). -* В конфигурации внешних словарей, пустое значение `null_value` интерпретируется, как значение типа данных по-умоланию. -* Поддержка функций `intDiv`, `intDivOrZero` для `Decimal`. -* Поддержка типов `Date`, `DateTime`, `UUID`, `Decimal` в качестве ключа для агрегатной функции `sumMap`. -* Поддержка типа данных `Decimal` во внешних словарях. -* Поддержка типа данных `Decimal` в таблицах типа `SummingMergeTree`. -* Добавлена специализация для `UUID` в функции `if`. -* Уменьшено количество системных вызовов `open`, `close` при чтении из таблиц семейства `MergeTree`. -* Возможность выполнения запроса `TRUNCATE TABLE` на любой реплике (запрос пробрасывается на реплику-лидера). [Kirill Shvakov]() +* Логгирование потребления памяти запросом при превышении очередной отметки целого числа гигабайт. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* Добавлен режим совместимости для случая, когда клиентская библиотека, работающая по Native протоколу, по ошибке отправляет меньшее количество столбцов, чем сервер ожидает для запроса INSERT. Такой сценарий был возможен при использовании библиотеки clickhouse-cpp. Ранее этот сценарий приводил к падению сервера. [#3171](https://github.com/yandex/ClickHouse/pull/3171) +* В `clickhouse-copier`, в задаваемом пользователем выражении WHERE, появилась возможность использовать алиас `partition_key` (для дополнительной фильтрации по партициям исходной таблицы). Это полезно, если схема партиционирования изменяется при копировании, но изменяется незначительно. [#3166](https://github.com/yandex/ClickHouse/pull/3166) +* Рабочий поток движка `Kafka` перенесён в фоновый пул потоков для того, чтобы автоматически уменьшать скорость чтения данных при большой нагрузке. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* Поддержка чтения значений типа `Tuple` и `Nested` структур как `struct` в формате `Cap'n'Proto` [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3216). +* В список доменов верхнего уровня для функции `firstSignificantSubdomain` добавлен домен `biz` [decaseal](https://github.com/yandex/ClickHouse/pull/3219). +* В конфигурации внешних словарей, пустое значение `null_value` интерпретируется, как значение типа данных по-умоланию. [#3330](https://github.com/yandex/ClickHouse/pull/3330) +* Поддержка функций `intDiv`, `intDivOrZero` для `Decimal`. [b48402e8](https://github.com/yandex/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) +* Поддержка типов `Date`, `DateTime`, `UUID`, `Decimal` в качестве ключа для агрегатной функции `sumMap`. [#3281](https://github.com/yandex/ClickHouse/pull/3281) +* Поддержка типа данных `Decimal` во внешних словарях. [#3324](https://github.com/yandex/ClickHouse/pull/3324) +* Поддержка типа данных `Decimal` в таблицах типа `SummingMergeTree`. [#3348](https://github.com/yandex/ClickHouse/pull/3348) +* Добавлена специализация для `UUID` в функции `if`. [#3366](https://github.com/yandex/ClickHouse/pull/3366) +* Уменьшено количество системных вызовов `open`, `close` при чтении из таблиц семейства `MergeTree` [#3283](https://github.com/yandex/ClickHouse/pull/3283). +* Возможность выполнения запроса `TRUNCATE TABLE` на любой реплике (запрос пробрасывается на реплику-лидера). [Kirill Shvakov](https://github.com/yandex/ClickHouse/pull/3375) ### Исправление ошибок: -* Исправлена ошибка в работе таблиц типа `Dictionary` для словарей типа `range_hashed`. Ошибка возникла в версии 18.12.17. -* Исправлена ошибка при загрузке словарей типа `range_hashed` (сообщение `Unsupported type Nullable(...)`). Ошибка возникла в версии 18.12.17. -* Исправлена некорректная работа функции `pointInPolygon` из-за накопления погрешности при вычислениях для полигонов с большим количеством близко расположенных вершин. -* Если после слияния кусков данных, у результирующего куска чексумма отличается от результата того же слияния на другой реплике, то результат слияния удаляется, и вместо этого кусок скачивается с другой реплики (это правильное поведение). Но после скачивания куска, он не мог добавиться в рабочий набор из-за ошибки, что кусок уже существует (так как кусок после слияния удалялся не сразу, а с задержкой). Это приводило к циклическим попыткам скачивания одних и тех же данных. -* Исправлен некорректный учёт общего потребления оперативной памяти запросами (что приводило к неправильной работе настройки `max_memory_usage_for_all_queries` и неправильному значению метрики `MemoryTracking`). Ошибка возникла в версии 18.12.13. [Marek Vavruša]() -* Исправлена работоспособность запросов `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Ошибка возникла в версии 18.12.13. -* Исправлена лишняя подготовка структуры данных для `JOIN` на сервере-инициаторе запроса, если `JOIN` выполняется только на удалённых серверах. -* Исправлены ошибки в движке `Kafka`: неработоспособность после исключения при начале чтения данных; блокировка при завершении [Marek Vavruša](). -* Для таблиц `Kafka` не передавался опциональный параметр `schema` (схема формата `Cap'n'Proto`). [Vojtech Splichal] -* Если ансамбль серверов ZooKeeper содержит серверы, которые принимают соединение, но сразу же разрывают его вместо ответа на рукопожатие, то ClickHouse выбирает для соединения другой сервер. Ранее в этом случае возникала ошибка `Cannot read all data. Bytes read: 0. Bytes expected: 4.` и сервер не мог стартовать. -* Если ансамбль серверов ZooKeeper содержит серверы, для которых DNS запрос возвращает ошибку, то такие серверы пропускаются. -* Исправлено преобразование типов между `Date` и `DateTime` при вставке данных в формате `VALUES` (в случае, когда `input_format_values_interpret_expressions = 1`). Ранее преобразование производилось между числовым значением количества дней с начала unix эпохи и unix timestamp, что приводило к неожиданным результатам. -* Исправление преобразования типов между `Decimal` и целыми числами. -* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang]() -* Настройка `enable_optimize_predicate_expression` выключена по-умолчанию. -* Исправлена ошибка парсинга формата CSV с числами с плавающей запятой, если используется разделитель CSV не по-умолчанию, такой как например, `;`. -* Испоавлена функция `arrayCumSumNonNegative` (она не накапливает отрицательные значения, если аккумулятор становится меньше нуля). -* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц при использовании `PREWHERE`. -* Исправления ошибок в запросе ALTER UPDATE. -* Исправления ошибок в табличной функции `odbc`, которые возникли в версии 18.12. -* Исправлена работа агрегатных функций с комбинаторами `StateArray`. -* Исправлено падение при делении значения типа `Decimal` на ноль. -* Исправлен вывод типов для операций с использованием аргументов типа `Decimal` и целых чисел. -* Исправлен segfault при `GROUP BY` по `Decimal128`. -* Настройка `log_query_threads` (логгирование информации о каждом потоке исполнения запроса) теперь имеет эффект только если настройка `log_queries` (логгирование информации о запросах) выставлена в 1. Так как настройка `log_query_threads` включена по-умолчанию, ранее информация о потоках логгировалась даже если логгирование запросов выключено. -* Исправлена ошибка в распределённой работе агрегатной функции quantiles (сообщение об ошибке вида `Not found column quantile...`). -* Исправлена проблема совместимости при одновременной работе на кластере серверов версии 18.12.17 и более старых, приводящая к тому, что при распределённых запросах с GROUP BY по ключам одновременно фиксированной и не фиксированной длины, при условии, что количество данных в процессе агрегации большое, могли возвращаться не до конца агрегированные данные (одни и те же ключи агрегации в двух разных строках). -* Исправлена обработка подстановок в `clickhouse-performance-test`, если запрос содержит только часть из объявленных в тесте подстановок. -* Исправлена ошибка при использовании `FINAL` совместно с `PREWHERE`. -* Исправлена ошибка при использовании `PREWHERE` над столбцами, добавленными при `ALTER`. -* Добавлена проверка отсутствия `arrayJoin` для `DEFAULT`, `MATERIALIZED` выражений. Ранее наличие `arrayJoin` приводило к ошибке при вставке данных. -* Добавлена проверка отсутствия `arrayJoin` в секции `PREWHERE`. Ранее это приводило к сообщениям вида `Size ... doesn't match` или `Unknown compression method` при выполнении запросов. -* Исправлен segfault, который мог возникать в редких случаях после оптимизации - замены цепочек AND из равенства выражения константам на соответствующее выражение IN. [liuyimin](). -* Мелкие исправления `clickhouse-benchmark`: ранее информация о клиенте не передавалась на сервер; более корректный подсчёт числа выполненных запросов при завершении работы и для ограничения числа итераций. +* Исправлена ошибка в работе таблиц типа `Dictionary` для словарей типа `range_hashed`. Ошибка возникла в версии 18.12.17. [#1702](https://github.com/yandex/ClickHouse/pull/1702) +* Исправлена ошибка при загрузке словарей типа `range_hashed` (сообщение `Unsupported type Nullable(...)`). Ошибка возникла в версии 18.12.17. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Исправлена некорректная работа функции `pointInPolygon` из-за накопления погрешности при вычислениях для полигонов с большим количеством близко расположенных вершин. [#3331](https://github.com/yandex/ClickHouse/pull/3331) [#3341](https://github.com/yandex/ClickHouse/pull/3341) +* Если после слияния кусков данных, у результирующего куска чексумма отличается от результата того же слияния на другой реплике, то результат слияния удаляется, и вместо этого кусок скачивается с другой реплики (это правильное поведение). Но после скачивания куска, он не мог добавиться в рабочий набор из-за ошибки, что кусок уже существует (так как кусок после слияния удалялся не сразу, а с задержкой). Это приводило к циклическим попыткам скачивания одних и тех же данных. [#3194](https://github.com/yandex/ClickHouse/pull/3194) +* Исправлен некорректный учёт общего потребления оперативной памяти запросами (что приводило к неправильной работе настройки `max_memory_usage_for_all_queries` и неправильному значению метрики `MemoryTracking`). Ошибка возникла в версии 18.12.13. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3344) +* Исправлена работоспособность запросов `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Ошибка возникла в версии 18.12.13. [#3247](https://github.com/yandex/ClickHouse/pull/3247) +* Исправлена лишняя подготовка структуры данных для `JOIN` на сервере-инициаторе запроса, если `JOIN` выполняется только на удалённых серверах. [#3340](https://github.com/yandex/ClickHouse/pull/3340) +* Исправлены ошибки в движке `Kafka`: неработоспособность после исключения при начале чтения данных; блокировка при завершении [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* Для таблиц `Kafka` не передавался опциональный параметр `schema` (схема формата `Cap'n'Proto`). [Vojtech Splichal](https://github.com/yandex/ClickHouse/pull/3150) +* Если ансамбль серверов ZooKeeper содержит серверы, которые принимают соединение, но сразу же разрывают его вместо ответа на рукопожатие, то ClickHouse выбирает для соединения другой сервер. Ранее в этом случае возникала ошибка `Cannot read all data. Bytes read: 0. Bytes expected: 4.` и сервер не мог стартовать. [8218cf3a](https://github.com/yandex/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +* Если ансамбль серверов ZooKeeper содержит серверы, для которых DNS запрос возвращает ошибку, то такие серверы пропускаются. [17b8e209](https://github.com/yandex/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) +* Исправлено преобразование типов между `Date` и `DateTime` при вставке данных в формате `VALUES` (в случае, когда `input_format_values_interpret_expressions = 1`). Ранее преобразование производилось между числовым значением количества дней с начала unix эпохи и unix timestamp, что приводило к неожиданным результатам. [#3229](https://github.com/yandex/ClickHouse/pull/3229) +* Исправление преобразования типов между `Decimal` и целыми числами. [#3211](https://github.com/yandex/ClickHouse/pull/3211) +* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3231) +* Исправлена ошибка парсинга формата CSV с числами с плавающей запятой, если используется разделитель CSV не по-умолчанию, такой как например, `;` [#3155](https://github.com/yandex/ClickHouse/pull/3155). +* Исправлена функция `arrayCumSumNonNegative` (она не накапливает отрицательные значения, если аккумулятор становится меньше нуля). [Aleksey Studnev](https://github.com/yandex/ClickHouse/pull/3163) +* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц при использовании `PREWHERE`. [#3165](https://github.com/yandex/ClickHouse/pull/3165) +* Исправления ошибок в запросе `ALTER UPDATE`. +* Исправления ошибок в табличной функции `odbc`, которые возникли в версии 18.12. [#3197](https://github.com/yandex/ClickHouse/pull/3197) +* Исправлена работа агрегатных функций с комбинаторами `StateArray`. [#3188](https://github.com/yandex/ClickHouse/pull/3188) +* Исправлено падение при делении значения типа `Decimal` на ноль. [69dd6609](https://github.com/yandex/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) +* Исправлен вывод типов для операций с использованием аргументов типа `Decimal` и целых чисел. [#3224](https://github.com/yandex/ClickHouse/pull/3224) +* Исправлен segfault при `GROUP BY` по `Decimal128`. [3359ba06](https://github.com/yandex/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) +* Настройка `log_query_threads` (логгирование информации о каждом потоке исполнения запроса) теперь имеет эффект только если настройка `log_queries` (логгирование информации о запросах) выставлена в 1. Так как настройка `log_query_threads` включена по-умолчанию, ранее информация о потоках логгировалась даже если логгирование запросов выключено. [#3241](https://github.com/yandex/ClickHouse/pull/3241) +* Исправлена ошибка в распределённой работе агрегатной функции quantiles (сообщение об ошибке вида `Not found column quantile...`). [292a8855](https://github.com/yandex/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) +* Исправлена проблема совместимости при одновременной работе на кластере серверов версии 18.12.17 и более старых, приводящая к тому, что при распределённых запросах с GROUP BY по ключам одновременно фиксированной и не фиксированной длины, при условии, что количество данных в процессе агрегации большое, могли возвращаться не до конца агрегированные данные (одни и те же ключи агрегации в двух разных строках). [#3254](https://github.com/yandex/ClickHouse/pull/3254) +* Исправлена обработка подстановок в `clickhouse-performance-test`, если запрос содержит только часть из объявленных в тесте подстановок. [#3263](https://github.com/yandex/ClickHouse/pull/3263) +* Исправлена ошибка при использовании `FINAL` совместно с `PREWHERE`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Исправлена ошибка при использовании `PREWHERE` над столбцами, добавленными при `ALTER`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Добавлена проверка отсутствия `arrayJoin` для `DEFAULT`, `MATERIALIZED` выражений. Ранее наличие `arrayJoin` приводило к ошибке при вставке данных. [#3337](https://github.com/yandex/ClickHouse/pull/3337) +* Добавлена проверка отсутствия `arrayJoin` в секции `PREWHERE`. Ранее это приводило к сообщениям вида `Size ... doesn't match` или `Unknown compression method` при выполнении запросов. [#3357](https://github.com/yandex/ClickHouse/pull/3357) +* Исправлен segfault, который мог возникать в редких случаях после оптимизации - замены цепочек AND из равенства выражения константам на соответствующее выражение IN. [liuyimin-bytedance](https://github.com/yandex/ClickHouse/pull/3339). +* Мелкие исправления `clickhouse-benchmark`: ранее информация о клиенте не передавалась на сервер; более корректный подсчёт числа выполненных запросов при завершении работы и для ограничения числа итераций. [#3351](https://github.com/yandex/ClickHouse/pull/3351) [#3352](https://github.com/yandex/ClickHouse/pull/3352) ### Обратно несовместимые изменения: -* Удалена настройка `allow_experimental_decimal_type`. Тип данных `Decimal` доступен для использования по-умолчанию. +* Удалена настройка `allow_experimental_decimal_type`. Тип данных `Decimal` доступен для использования по-умолчанию. [#3329](https://github.com/yandex/ClickHouse/pull/3329) ## ClickHouse release 18.12.17, 2018-09-16 ### Новые возможности: -* `invalidate_query` (возможность задать запрос для проверки необходимости обновления внешнего словаря) реализована для источника `clickhouse`. -* Добавлена возможность использования типов данных `UInt*`, `Int*`, `DateTime` (наравне с типом `Date`) в качестве ключа внешнего словаря типа `range_hashed`, определяющего границы диапазонов. Возможность использования NULL в качестве обозначения открытого диапазона. [Vasily Nemkov]() -* Для типа `Decimal` добавлена поддержка агрегатных функций `var*`, `stddev*`. -* Для типа `Decimal` добавлена поддержка математических функций (`exp`, `sin` и т. п.) -* В таблицу `system.part_log` добавлен столбец `partition_id`. +* `invalidate_query` (возможность задать запрос для проверки необходимости обновления внешнего словаря) реализована для источника `clickhouse`. [#3126](https://github.com/yandex/ClickHouse/pull/3126) +* Добавлена возможность использования типов данных `UInt*`, `Int*`, `DateTime` (наравне с типом `Date`) в качестве ключа внешнего словаря типа `range_hashed`, определяющего границы диапазонов. Возможность использования `NULL` в качестве обозначения открытого диапазона. [Vasily Nemkov](https://github.com/yandex/ClickHouse/pull/3123) +* Для типа `Decimal` добавлена поддержка агрегатных функций `var*`, `stddev*`. [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* Для типа `Decimal` добавлена поддержка математических функций (`exp`, `sin` и т. п.) [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* В таблицу `system.part_log` добавлен столбец `partition_id`. [#3089](https://github.com/yandex/ClickHouse/pull/3089) ### Исправление ошибок: -* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц. [Winter Zhang]() -* Исправлена несовместимость (лишняя зависимость от версии `glibc`), приводящая к невозможности запуска ClickHouse на `Ubuntu Precise` и более старых. Несовместимость возникла в версии 18.12.13. -* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang]() -* Исправлено незначительное нарушение обратной совместимости, проявляющееся при одновременной работе на кластере реплик версий до 18.12.13 и создании новой реплики таблицы на сервере более новой версии (выдаётся сообщение `Can not clone replica, because the ... updated to new ClickHouse version`, что полностью логично, но не должно было происходить). +* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3159) +* Исправлена несовместимость (лишняя зависимость от версии `glibc`), приводящая к невозможности запуска ClickHouse на `Ubuntu Precise` и более старых. Несовместимость возникла в версии 18.12.13. [#3130](https://github.com/yandex/ClickHouse/pull/3130) +* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) +* Исправлено незначительное нарушение обратной совместимости, проявляющееся при одновременной работе на кластере реплик версий до 18.12.13 и создании новой реплики таблицы на сервере более новой версии (выдаётся сообщение `Can not clone replica, because the ... updated to new ClickHouse version`, что полностью логично, но не должно было происходить). [#3122](https://github.com/yandex/ClickHouse/pull/3122) ### Обратно несовместимые изменения: -* Настройка `enable_optimize_predicate_expression` включена по-умолчанию, что конечно очень оптимистично. При возникновении ошибок анализа запроса, связанных с поиском имён столбцов, следует выставить `enable_optimize_predicate_expression` в 0. [Winter Zhang]() +* Настройка `enable_optimize_predicate_expression` включена по-умолчанию, что конечно очень оптимистично. При возникновении ошибок анализа запроса, связанных с поиском имён столбцов, следует выставить `enable_optimize_predicate_expression` в 0. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) ## ClickHouse release 18.12.14, 2018-09-13 diff --git a/README.md b/README.md index e5ca7ce2ec5..a488c55643c 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,11 @@ ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time. +🎤🥂 **ClickHouse Meetup in [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/)** 🍰🔥🐻 + ## Useful Links * [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page. * [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster. * [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information. * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any. - -## Upcoming Meetups - -* [Beijing on October 28](http://www.clickhouse.com.cn/topic/5ba0e3f99d28dfde2ddc62a1) -* [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/) diff --git a/cmake/find_odbc.cmake b/cmake/find_odbc.cmake index 95acf40b2b4..d89e3b532d8 100644 --- a/cmake/find_odbc.cmake +++ b/cmake/find_odbc.cmake @@ -86,4 +86,4 @@ if (ENABLE_ODBC) endif () endif () -message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") +message (STATUS "Using odbc=${ODBC_FOUND}: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index 79d5930bc9c..d8468e5306d 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -116,10 +116,10 @@ endif () if (Poco_MongoDB_LIBRARY) set (USE_POCO_MONGODB 1) endif () -if (Poco_DataODBC_LIBRARY) +if (Poco_DataODBC_LIBRARY AND ODBC_FOUND) set (USE_POCO_DATAODBC 1) endif () -if (Poco_SQLODBC_LIBRARY) +if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND) set (USE_POCO_SQLODBC 1) endif () diff --git a/contrib/ssl b/contrib/ssl index de02224a42c..919f6f1331d 160000 --- a/contrib/ssl +++ b/contrib/ssl @@ -1 +1 @@ -Subproject commit de02224a42c69e3d8c9112c82018816f821878d0 +Subproject commit 919f6f1331d500bfdd26f8bbbf88e92c0119879b diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 0cb5824e1fc..91171eea685 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -6,18 +6,12 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_vectorclass.cmake) set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h) set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h) -set (CONFIG_BUILD ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_build.cpp) include (cmake/version.cmake) message (STATUS "Will build ${VERSION_FULL}") configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config.h.in ${CONFIG_COMMON}) configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_version.h.in ${CONFIG_VERSION}) -get_property (BUILD_COMPILE_DEFINITIONS DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY COMPILE_DEFINITIONS) -get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) -string (TIMESTAMP BUILD_DATE "%Y-%m-%d" UTC) -configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_build.cpp.in ${CONFIG_BUILD}) - if (NOT MSVC) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") endif () diff --git a/dbms/programs/client/ConnectionParameters.h b/dbms/programs/client/ConnectionParameters.h index ee381bed2e7..68bc3728349 100644 --- a/dbms/programs/client/ConnectionParameters.h +++ b/dbms/programs/client/ConnectionParameters.h @@ -76,7 +76,8 @@ struct ConnectionParameters timeouts = ConnectionTimeouts( Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0)); + Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("tcp_keep_alive_timeout", 0), 0)); } }; diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 1a2f321fa00..9d9324b9a3e 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -63,6 +63,8 @@ namespace ErrorCodes extern const int TOO_BIG_AST; extern const int UNEXPECTED_AST_STRUCTURE; + extern const int SYNTAX_ERROR; + extern const int UNKNOWN_TABLE; extern const int UNKNOWN_FUNCTION; extern const int UNKNOWN_IDENTIFIER; @@ -109,6 +111,8 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti exception_code == ErrorCodes::TOO_BIG_AST || exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE) return HTTPResponse::HTTP_BAD_REQUEST; + else if (exception_code == ErrorCodes::SYNTAX_ERROR) + return HTTPResponse::HTTP_BAD_REQUEST; else if (exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION || exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 79c8e08fd14..5c9b3a2d86d 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -121,6 +121,9 @@ void TCPHandler::runImpl() while (1) { + /// Restore context of request. + query_context = connection_context; + /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (!static_cast(*in).poll(global_settings.poll_interval * 1000000) && !server.isCancelled()) ; @@ -145,9 +148,6 @@ void TCPHandler::runImpl() try { - /// Restore context of request. - query_context = connection_context; - /// If a user passed query-local timeouts, reset socket to initial state at the end of the query SCOPE_EXIT({state.timeout_setter.reset();}); diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp index 4dcdaf790e5..5e969c802e5 100644 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 727fa6fb5a0..ce6246fba3a 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -77,6 +77,17 @@ void Connection::connect() socket->setReceiveTimeout(timeouts.receive_timeout); socket->setSendTimeout(timeouts.send_timeout); socket->setNoDelay(true); + if (timeouts.tcp_keep_alive_timeout.totalSeconds()) + { + socket->setKeepAlive(true); + socket->setOption(IPPROTO_TCP, +#if defined(TCP_KEEPALIVE) + TCP_KEEPALIVE +#else + TCP_KEEPIDLE // __APPLE__ +#endif + , timeouts.tcp_keep_alive_timeout); + } in = std::make_shared(*socket); out = std::make_shared(*socket); diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index 8cefc16500b..eebf9b75a35 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -462,6 +462,8 @@ XMLDocumentPtr ConfigProcessor::processConfig( std::string include_from_path; if (node) { + /// if we include_from env or zk. + doIncludesRecursive(config, nullptr, node, zk_node_cache, contributing_zk_paths); include_from_path = node->innerText(); } else diff --git a/dbms/src/Common/config_build.h b/dbms/src/Common/config_build.h deleted file mode 100644 index 93e9ba35a8d..00000000000 --- a/dbms/src/Common/config_build.h +++ /dev/null @@ -1,4 +0,0 @@ -#pragma once -#include - -extern const char * auto_config_build[]; diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index f1e68a6a0c1..ff5fc75f1c4 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -33,9 +33,6 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const if (Protocol::Server::Data == packet.type) { header = packet.block; - - if (!header) - throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); break; } else if (Protocol::Server::Exception == packet.type) @@ -58,7 +55,8 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const void RemoteBlockOutputStream::write(const Block & block) { - assertBlocksHaveEqualStructure(block, header, "RemoteBlockOutputStream"); + if (header) + assertBlocksHaveEqualStructure(block, header, "RemoteBlockOutputStream"); try { diff --git a/dbms/src/Dictionaries/DictionarySourceFactory.cpp b/dbms/src/Dictionaries/DictionarySourceFactory.cpp index bb778fbce62..1da2268c1e5 100644 --- a/dbms/src/Dictionaries/DictionarySourceFactory.cpp +++ b/dbms/src/Dictionaries/DictionarySourceFactory.cpp @@ -156,7 +156,7 @@ DictionarySourcePtr DictionarySourceFactory::create( { #if USE_POCO_SQLODBC || USE_POCO_DATAODBC const auto & global_config = context.getConfigRef(); - BridgeHelperPtr bridge = std::make_shared>(global_config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".odbc.connection_string")); + BridgeHelperPtr bridge = std::make_shared>(global_config, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); return std::make_unique(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge); #else throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.", @@ -167,7 +167,7 @@ DictionarySourcePtr DictionarySourceFactory::create( { throw Exception{"Dictionary source of type `jdbc` is disabled until consistent support for nullable fields.", ErrorCodes::SUPPORT_IS_DISABLED}; -// BridgeHelperPtr bridge = std::make_shared>(config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".connection_string")); +// BridgeHelperPtr bridge = std::make_shared>(config, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".connection_string")); // return std::make_unique(dict_struct, config, config_prefix + ".jdbc", sample_block, context, bridge); } else if ("executable" == source_type) diff --git a/dbms/src/Formats/ValuesRowInputStream.cpp b/dbms/src/Formats/ValuesRowInputStream.cpp index 559ac658a6a..13d013a8ac9 100644 --- a/dbms/src/Formats/ValuesRowInputStream.cpp +++ b/dbms/src/Formats/ValuesRowInputStream.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,20 @@ namespace ErrorCodes } +bool is_array_type_compatible(const DataTypeArray & type, const Field & value) +{ + if (type.getNestedType()->isNullable()) + return true; + + const Array & array = DB::get(value); + size_t size = array.size(); + for (size_t i = 0; i < size; ++i) + if (array[i].isNull()) + return false; + + return true; +} + ValuesRowInputStream::ValuesRowInputStream(ReadBuffer & istr_, const Block & header_, const Context & context_, const FormatSettings & format_settings) : istr(istr_), header(header_), context(std::make_unique(context_)), format_settings(format_settings) { @@ -116,14 +131,15 @@ bool ValuesRowInputStream::read(MutableColumns & columns) std::pair value_raw = evaluateConstantExpression(ast, *context); Field value = convertFieldToType(value_raw.first, type, value_raw.second.get()); - if (value.isNull()) + const auto * array_type = typeid_cast(&type); + + /// Check that we are indeed allowed to insert a NULL. + if ((value.isNull() && !type.isNullable()) || (array_type && !is_array_type_compatible(*array_type, value))) { - /// Check that we are indeed allowed to insert a NULL. - if (!type.isNullable()) - throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value) - + ", that is out of range of type " + type.getName() - + ", at: " + String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)), - ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE}; + throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value) + + ", that is out of range of type " + type.getName() + + ", at: " + String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)), + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE}; } columns[i]->insert(value); diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index 9335e5eee57..f9802cd5178 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -784,6 +784,9 @@ public: } else if constexpr (to_decimal) { + if (!arguments[1].column) + throw Exception("Second argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN); + UInt64 scale = extractToDecimalScale(arguments[1]); if constexpr (std::is_same_v) diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index 8d6142bc287..e0d1081246a 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -288,7 +288,10 @@ bool PreparedFunctionImpl::defaultImplementationForConstantArguments(Block & blo const ColumnWithTypeAndName & column = block.getByPosition(args[arg_num]); if (arguments_to_remain_constants.end() != std::find(arguments_to_remain_constants.begin(), arguments_to_remain_constants.end(), arg_num)) - temporary_block.insert(column); + if (column.column->empty()) + temporary_block.insert({column.column->cloneResized(1), column.type, column.name}); + else + temporary_block.insert(column); else { have_converted_columns = true; @@ -311,7 +314,15 @@ bool PreparedFunctionImpl::defaultImplementationForConstantArguments(Block & blo executeWithoutLowCardinalityColumns(temporary_block, temporary_argument_numbers, arguments_size, temporary_block.rows()); - block.getByPosition(result).column = ColumnConst::create(temporary_block.getByPosition(arguments_size).column, input_rows_count); + ColumnPtr result_column; + /// extremely rare case, when we have function with completely const arguments + /// but some of them produced by non isDeterministic function + if (temporary_block.getByPosition(arguments_size).column->size() > 1) + result_column = temporary_block.getByPosition(arguments_size).column->cloneResized(1); + else + result_column = temporary_block.getByPosition(arguments_size).column; + + block.getByPosition(result).column = ColumnConst::create(result_column, input_rows_count); return true; } diff --git a/dbms/src/IO/ConnectionTimeouts.h b/dbms/src/IO/ConnectionTimeouts.h index dc2d58e5d04..fed1ec3f0e4 100644 --- a/dbms/src/IO/ConnectionTimeouts.h +++ b/dbms/src/IO/ConnectionTimeouts.h @@ -11,6 +11,7 @@ struct ConnectionTimeouts Poco::Timespan connection_timeout; Poco::Timespan send_timeout; Poco::Timespan receive_timeout; + Poco::Timespan tcp_keep_alive_timeout; ConnectionTimeouts() = default; @@ -19,7 +20,19 @@ struct ConnectionTimeouts const Poco::Timespan & receive_timeout_) : connection_timeout(connection_timeout_), send_timeout(send_timeout_), - receive_timeout(receive_timeout_) + receive_timeout(receive_timeout_), + tcp_keep_alive_timeout(0) + { + } + + ConnectionTimeouts(const Poco::Timespan & connection_timeout_, + const Poco::Timespan & send_timeout_, + const Poco::Timespan & receive_timeout_, + const Poco::Timespan & tcp_keep_alive_timeout_) + : connection_timeout(connection_timeout_), + send_timeout(send_timeout_), + receive_timeout(receive_timeout_), + tcp_keep_alive_timeout(tcp_keep_alive_timeout_) { } @@ -35,19 +48,20 @@ struct ConnectionTimeouts { return ConnectionTimeouts(saturate(connection_timeout, limit), saturate(send_timeout, limit), - saturate(receive_timeout, limit)); + saturate(receive_timeout, limit), + saturate(tcp_keep_alive_timeout, limit)); } /// Timeouts for the case when we have just single attempt to connect. static ConnectionTimeouts getTCPTimeoutsWithoutFailover(const Settings & settings) { - return ConnectionTimeouts(settings.connect_timeout, settings.send_timeout, settings.receive_timeout); + return ConnectionTimeouts(settings.connect_timeout, settings.send_timeout, settings.receive_timeout, settings.tcp_keep_alive_timeout); } /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings) { - return ConnectionTimeouts(settings.connect_timeout_with_failover_ms, settings.send_timeout, settings.receive_timeout); + return ConnectionTimeouts(settings.connect_timeout_with_failover_ms, settings.send_timeout, settings.receive_timeout, settings.tcp_keep_alive_timeout); } static ConnectionTimeouts getHTTPTimeouts(const Settings & settings) diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index 29d8f190fbf..8eebf869303 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -568,8 +568,8 @@ void ActionsVisitor::makeSet(const ASTFunction * node, const Block & sample_bloc /// and the table has the type Set (a previously prepared set). if (identifier) { - auto database_table = getDatabaseAndTableNameFromIdentifier(*identifier); - StoragePtr table = context.tryGetTable(database_table.first, database_table.second); + DatabaseAndTableWithAlias database_table(*identifier); + StoragePtr table = context.tryGetTable(database_table.database, database_table.table); if (table) { diff --git a/dbms/src/Interpreters/AddDefaultDatabaseVisitor.h b/dbms/src/Interpreters/AddDefaultDatabaseVisitor.h new file mode 100644 index 00000000000..a9aed99c10f --- /dev/null +++ b/dbms/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Visits AST nodes, add default database to DDLs if not set. +class AddDefaultDatabaseVisitor +{ +public: + AddDefaultDatabaseVisitor(const String & default_database_) + : default_database(default_database_) + {} + + void visit(ASTPtr & ast) const + { + visitChildren(ast); + + if (!tryVisit(ast) && + !tryVisit(ast)) + {} + } + +private: + const String default_database; + + void visit(ASTQueryWithTableAndOutput * node, ASTPtr &) const + { + if (node->database.empty()) + node->database = default_database; + } + + void visit(ASTRenameQuery * node, ASTPtr &) const + { + for (ASTRenameQuery::Element & elem : node->elements) + { + if (elem.from.database.empty()) + elem.from.database = default_database; + if (elem.to.database.empty()) + elem.to.database = default_database; + } + } + + void visitChildren(ASTPtr & ast) const + { + for (auto & child : ast->children) + visit(child); + } + + template + bool tryVisit(ASTPtr & ast) const + { + if (T * t = dynamic_cast(ast.get())) + { + visit(t, ast); + return true; + } + return false; + } +}; + +} diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 8fa02c91f6a..02e48a838bd 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int UNKNOWN_ELEMENT_IN_CONFIG; extern const int INVALID_CONFIG_PARAMETER; extern const int UNKNOWN_FORMAT_VERSION; @@ -1135,7 +1137,7 @@ private: }; -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, const NameSet & query_databases) +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, NameSet && query_databases) { /// Remove FORMAT and INTO OUTFILE if exists ASTPtr query_ptr = query_ptr_->clone(); @@ -1163,30 +1165,56 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont ClusterPtr cluster = context.getCluster(query->cluster); DDLWorker & ddl_worker = context.getDDLWorker(); - DDLLogEntry entry; - entry.query = queryToString(query_ptr); - entry.initiator = ddl_worker.getCommonHostID(); - /// Check database access rights, assume that all servers have the same users config - NameSet databases_to_check_access_rights; + NameSet databases_to_access; + const String & current_database = context.getCurrentDatabase(); Cluster::AddressesWithFailover shards = cluster->getShardsAddresses(); + std::vector hosts; + bool use_shard_default_db = false; + bool use_local_default_db = false; for (const auto & shard : shards) { for (const auto & addr : shard) { - entry.hosts.emplace_back(addr); + hosts.emplace_back(addr); - /// Expand empty database name to shards' default database name + /// Expand empty database name to shards' default (o current) database name for (const String & database : query_databases) - databases_to_check_access_rights.emplace(database.empty() ? addr.default_database : database); + { + if (database.empty()) + { + bool has_shard_default_db = !addr.default_database.empty(); + use_shard_default_db |= has_shard_default_db; + use_local_default_db |= !has_shard_default_db; + databases_to_access.emplace(has_shard_default_db ? addr.default_database : current_database ); + } + else + databases_to_access.emplace(database); + } } } - for (const String & database : databases_to_check_access_rights) - context.checkDatabaseAccessRights(database.empty() ? context.getCurrentDatabase() : database); + if (use_shard_default_db && use_local_default_db) + throw Exception("Mixed local default DB and shard default DB in DDL query", ErrorCodes::NOT_IMPLEMENTED); + if (databases_to_access.empty()) + throw Exception("No databases to access in distributed DDL query", ErrorCodes::LOGICAL_ERROR); + + for (const String & database : databases_to_access) + context.checkDatabaseAccessRights(database); + + if (use_local_default_db) + { + AddDefaultDatabaseVisitor visitor(current_database); + visitor.visit(query_ptr); + } + + DDLLogEntry entry; + entry.hosts = std::move(hosts); + entry.query = queryToString(query_ptr); + entry.initiator = ddl_worker.getCommonHostID(); String node_path = ddl_worker.enqueueQuery(entry); BlockIO io; diff --git a/dbms/src/Interpreters/DDLWorker.h b/dbms/src/Interpreters/DDLWorker.h index 2fba83b7356..d3872b8ac95 100644 --- a/dbms/src/Interpreters/DDLWorker.h +++ b/dbms/src/Interpreters/DDLWorker.h @@ -20,7 +20,7 @@ struct DDLTask; /// Pushes distributed DDL query to the queue -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const NameSet & query_databases); +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, NameSet && query_databases); class DDLWorker diff --git a/dbms/src/Interpreters/DatabaseAndTableWithAlias.cpp b/dbms/src/Interpreters/DatabaseAndTableWithAlias.cpp new file mode 100644 index 00000000000..df430ee0fbd --- /dev/null +++ b/dbms/src/Interpreters/DatabaseAndTableWithAlias.cpp @@ -0,0 +1,245 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Checks that ast is ASTIdentifier and remove num_qualifiers_to_strip components from left. +/// Example: 'database.table.name' -> (num_qualifiers_to_strip = 2) -> 'name'. +void stripIdentifier(DB::ASTPtr & ast, size_t num_qualifiers_to_strip) +{ + ASTIdentifier * identifier = typeid_cast(ast.get()); + + if (!identifier) + throw DB::Exception("ASTIdentifier expected for stripIdentifier", DB::ErrorCodes::LOGICAL_ERROR); + + if (num_qualifiers_to_strip) + { + size_t num_components = identifier->children.size(); + + /// plain column + if (num_components - num_qualifiers_to_strip == 1) + { + DB::String node_alias = identifier->tryGetAlias(); + ast = identifier->children.back(); + if (!node_alias.empty()) + ast->setAlias(node_alias); + } + else + /// nested column + { + identifier->children.erase(identifier->children.begin(), identifier->children.begin() + num_qualifiers_to_strip); + DB::String new_name; + for (const auto & child : identifier->children) + { + if (!new_name.empty()) + new_name += '.'; + new_name += static_cast(*child.get()).name; + } + identifier->name = new_name; + } + } +} + +/// Get the number of components of identifier which are correspond to 'alias.', 'table.' or 'databas.table.' from names. +size_t getNumComponentsToStripInOrderToTranslateQualifiedName(const ASTIdentifier & identifier, + const DatabaseAndTableWithAlias & names) +{ + size_t num_qualifiers_to_strip = 0; + + auto get_identifier_name = [](const ASTPtr & ast) { return static_cast(*ast).name; }; + + /// It is compound identifier + if (!identifier.children.empty()) + { + size_t num_components = identifier.children.size(); + + /// database.table.column + if (num_components >= 3 + && !names.database.empty() + && get_identifier_name(identifier.children[0]) == names.database + && get_identifier_name(identifier.children[1]) == names.table) + { + num_qualifiers_to_strip = 2; + } + + /// table.column or alias.column. If num_components > 2, it is like table.nested.column. + if (num_components >= 2 + && ((!names.table.empty() && get_identifier_name(identifier.children[0]) == names.table) + || (!names.alias.empty() && get_identifier_name(identifier.children[0]) == names.alias))) + { + num_qualifiers_to_strip = 1; + } + } + + return num_qualifiers_to_strip; +} + + +DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database) +{ + database = current_database; + table = identifier.name; + alias = identifier.tryGetAlias(); + + if (!identifier.children.empty()) + { + if (identifier.children.size() != 2) + throw Exception("Logical error: number of components in table expression not equal to two", ErrorCodes::LOGICAL_ERROR); + + const ASTIdentifier * db_identifier = typeid_cast(identifier.children[0].get()); + const ASTIdentifier * table_identifier = typeid_cast(identifier.children[1].get()); + if (!db_identifier || !table_identifier) + throw Exception("Logical error: identifiers expected", ErrorCodes::LOGICAL_ERROR); + + database = db_identifier->name; + table = table_identifier->name; + } +} + +DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database) +{ + if (table_expression.database_and_table_name) + { + const auto * identifier = typeid_cast(table_expression.database_and_table_name.get()); + if (!identifier) + throw Exception("Logical error: identifier expected", ErrorCodes::LOGICAL_ERROR); + + *this = DatabaseAndTableWithAlias(*identifier, current_database); + } + else if (table_expression.table_function) + alias = table_expression.table_function->tryGetAlias(); + else if (table_expression.subquery) + alias = table_expression.subquery->tryGetAlias(); + else + throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR); +} + +String DatabaseAndTableWithAlias::getQualifiedNamePrefix() const +{ + if (alias.empty() && table.empty()) + return ""; + + return (!alias.empty() ? alias : (database + '.' + table)) + '.'; +} + +void DatabaseAndTableWithAlias::makeQualifiedName(const ASTPtr & ast) const +{ + if (auto identifier = typeid_cast(ast.get())) + { + String prefix = getQualifiedNamePrefix(); + identifier->name.insert(identifier->name.begin(), prefix.begin(), prefix.end()); + + Names qualifiers; + if (!alias.empty()) + qualifiers.push_back(alias); + else + { + qualifiers.push_back(database); + qualifiers.push_back(table); + } + + for (const auto & qualifier : qualifiers) + identifier->children.emplace_back(std::make_shared(qualifier)); + } +} + +std::vector getSelectTablesExpression(const ASTSelectQuery & select_query) +{ + if (!select_query.tables) + return {}; + + std::vector tables_expression; + + for (const auto & child : select_query.tables->children) + { + ASTTablesInSelectQueryElement * tables_element = static_cast(child.get()); + + if (tables_element->table_expression) + tables_expression.emplace_back(static_cast(tables_element->table_expression.get())); + } + + return tables_expression; +} + +static const ASTTableExpression * getTableExpression(const ASTSelectQuery & select, size_t table_number) +{ + if (!select.tables) + return {}; + + ASTTablesInSelectQuery & tables_in_select_query = static_cast(*select.tables); + if (tables_in_select_query.children.size() <= table_number) + return {}; + + ASTTablesInSelectQueryElement & tables_element = + static_cast(*tables_in_select_query.children[table_number]); + + if (!tables_element.table_expression) + return {}; + + return static_cast(tables_element.table_expression.get()); +} + +std::vector getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database) +{ + std::vector tables_expression = getSelectTablesExpression(select_query); + + std::vector database_and_table_with_aliases; + database_and_table_with_aliases.reserve(tables_expression.size()); + + for (const auto & table_expression : tables_expression) + database_and_table_with_aliases.emplace_back(DatabaseAndTableWithAlias(*table_expression, current_database)); + + return database_and_table_with_aliases; +} + +std::optional getDatabaseAndTable(const ASTSelectQuery & select, size_t table_number) +{ + const ASTTableExpression * table_expression = getTableExpression(select, table_number); + if (!table_expression) + return {}; + + ASTPtr database_and_table_name = table_expression->database_and_table_name; + if (!database_and_table_name) + return {}; + + const ASTIdentifier * identifier = typeid_cast(database_and_table_name.get()); + if (!identifier) + return {}; + + return *identifier; +} + +ASTPtr getTableFunctionOrSubquery(const ASTSelectQuery & select, size_t table_number) +{ + const ASTTableExpression * table_expression = getTableExpression(select, table_number); + if (table_expression) + { +#if 1 /// TODO: It hides some logical error in InterpreterSelectQuery & distributed tables + if (table_expression->database_and_table_name) + { + if (table_expression->database_and_table_name->children.empty()) + return table_expression->database_and_table_name; + + if (table_expression->database_and_table_name->children.size() == 2) + return table_expression->database_and_table_name->children[1]; + } +#endif + if (table_expression->table_function) + return table_expression->table_function; + + if (table_expression->subquery) + return static_cast(table_expression->subquery.get())->children[0]; + } + + return nullptr; +} + +} diff --git a/dbms/src/Interpreters/evaluateQualified.h b/dbms/src/Interpreters/DatabaseAndTableWithAlias.h similarity index 50% rename from dbms/src/Interpreters/evaluateQualified.h rename to dbms/src/Interpreters/DatabaseAndTableWithAlias.h index 94833190d81..2444f037d8e 100644 --- a/dbms/src/Interpreters/evaluateQualified.h +++ b/dbms/src/Interpreters/DatabaseAndTableWithAlias.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace DB @@ -9,16 +10,21 @@ namespace DB class IAST; using ASTPtr = std::shared_ptr; +class ASTSelectQuery; class ASTIdentifier; struct ASTTableExpression; +/// Extracts database name (and/or alias) from table expression or identifier struct DatabaseAndTableWithAlias { String database; String table; String alias; + DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database = ""); + DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database); + /// "alias." or "database.table." if alias is empty String getQualifiedNamePrefix() const; @@ -28,12 +34,13 @@ struct DatabaseAndTableWithAlias void stripIdentifier(DB::ASTPtr & ast, size_t num_qualifiers_to_strip); -DatabaseAndTableWithAlias getTableNameWithAliasFromTableExpression(const ASTTableExpression & table_expression, - const String & current_database); - size_t getNumComponentsToStripInOrderToTranslateQualifiedName(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & names); -std::pair getDatabaseAndTableNameFromIdentifier(const ASTIdentifier & identifier); +std::vector getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database); +std::optional getDatabaseAndTable(const ASTSelectQuery & select, size_t table_number); + +std::vector getSelectTablesExpression(const ASTSelectQuery & select_query); +ASTPtr getTableFunctionOrSubquery(const ASTSelectQuery & select, size_t table_number); } diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index b84c007d790..1daf0bc3729 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -206,8 +206,14 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings) if (auto * prepared_function = dynamic_cast(function.get())) prepared_function->createLowCardinalityResultCache(settings.max_threads); + bool compile_expressions = false; +#if USE_EMBEDDED_COMPILER + compile_expressions = settings.compile_expressions; +#endif /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function. - if (all_const && function_base->isSuitableForConstantFolding()) + /// But if we compile expressions compiled version of this function maybe placed in cache, + /// so we don't want to unfold non deterministic functions + if (all_const && function_base->isSuitableForConstantFolding() && (!compile_expressions || function_base->isDeterministic())) { function->execute(sample_block, arguments, result_position, sample_block.rows()); @@ -671,7 +677,8 @@ void ExpressionActions::addImpl(ExpressionAction action, Names & new_names) new_names.push_back(action.result_name); new_names.insert(new_names.end(), action.array_joined_columns.begin(), action.array_joined_columns.end()); - if (action.type == ExpressionAction::APPLY_FUNCTION) + /// Compiled functions are custom functions and them don't need building + if (action.type == ExpressionAction::APPLY_FUNCTION && !action.is_function_compiled) { if (sample_block.has(action.result_name)) throw Exception("Column '" + action.result_name + "' already exists", ErrorCodes::DUPLICATE_COLUMN); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index f90868d7a36..baa8c5d8e91 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -59,7 +59,7 @@ #include #include #include -#include +#include #include #include @@ -172,36 +172,26 @@ ExpressionAnalyzer::ExpressionAnalyzer( if (!storage && select_query) { - auto select_database = select_query->database(); - auto select_table = select_query->table(); - - if (select_table - && !typeid_cast(select_table.get()) - && !typeid_cast(select_table.get())) - { - String database = select_database - ? typeid_cast(*select_database).name - : ""; - const String & table = typeid_cast(*select_table).name; - storage = context.tryGetTable(database, table); - } + if (auto db_and_table = getDatabaseAndTable(*select_query, 0)) + storage = context.tryGetTable(db_and_table->database, db_and_table->table); } - if (storage && source_columns.empty()) + if (storage) { auto physical_columns = storage->getColumns().getAllPhysical(); if (source_columns.empty()) source_columns.swap(physical_columns); else - { source_columns.insert(source_columns.end(), physical_columns.begin(), physical_columns.end()); - removeDuplicateColumns(source_columns); + + if (select_query) + { + const auto & storage_aliases = storage->getColumns().aliases; + source_columns.insert(source_columns.end(), storage_aliases.begin(), storage_aliases.end()); } } - else - removeDuplicateColumns(source_columns); - addAliasColumns(); + removeDuplicateColumns(source_columns); translateQualifiedNames(); @@ -275,55 +265,14 @@ bool ExpressionAnalyzer::isRemoteStorage() const } -static std::vector getTableExpressions(const ASTPtr & query) -{ - ASTSelectQuery * select_query = typeid_cast(query.get()); - - std::vector table_expressions; - - if (select_query && select_query->tables) - { - for (const auto & element : select_query->tables->children) - { - ASTTablesInSelectQueryElement & select_element = static_cast(*element); - - if (select_element.table_expression) - table_expressions.emplace_back(static_cast(*select_element.table_expression)); - } - } - - return table_expressions; -} - void ExpressionAnalyzer::translateQualifiedNames() { if (!select_query || !select_query->tables || select_query->tables->children.empty()) return; - std::vector tables; - std::vector tables_expression = getTableExpressions(query); + std::vector tables = getDatabaseAndTables(*select_query, context.getCurrentDatabase()); LogAST log; - - for (const auto & table_expression : tables_expression) - { - auto table = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase()); - - { /// debug print - size_t depth = 0; - DumpASTNode dump(table_expression, log.stream(), depth, "getTableNames"); - if (table_expression.database_and_table_name) - DumpASTNode(*table_expression.database_and_table_name, log.stream(), depth); - if (table_expression.table_function) - DumpASTNode(*table_expression.table_function, log.stream(), depth); - if (table_expression.subquery) - DumpASTNode(*table_expression.subquery, log.stream(), depth); - dump.print("getTableNameWithAlias", table.database + '.' + table.table + ' ' + table.alias); - } - - tables.emplace_back(table); - } - TranslateQualifiedNamesVisitor visitor(source_columns, tables, log.stream()); visitor.visit(query); } @@ -574,8 +523,8 @@ static NamesAndTypesList getNamesAndTypeListFromTableExpression(const ASTTableEx else if (table_expression.database_and_table_name) { const auto & identifier = static_cast(*table_expression.database_and_table_name); - auto database_table = getDatabaseAndTableNameFromIdentifier(identifier); - const auto & table = context.getTable(database_table.first, database_table.second); + DatabaseAndTableWithAlias database_table(identifier); + const auto & table = context.getTable(database_table.database, database_table.table); names_and_type_list = table->getSampleBlockNonMaterialized().getNamesAndTypesList(); } @@ -602,13 +551,13 @@ void ExpressionAnalyzer::normalizeTree() TableNamesAndColumnNames table_names_and_column_names; if (select_query && select_query->tables && !select_query->tables->children.empty()) { - std::vector tables_expression = getTableExpressions(query); + std::vector tables_expression = getSelectTablesExpression(*select_query); bool first = true; - for (const auto & table_expression : tables_expression) + for (const auto * table_expression : tables_expression) { - const auto table_name = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase()); - NamesAndTypesList names_and_types = getNamesAndTypeListFromTableExpression(table_expression, context); + DatabaseAndTableWithAlias table_name(*table_expression, context.getCurrentDatabase()); + NamesAndTypesList names_and_types = getNamesAndTypeListFromTableExpression(*table_expression, context); if (!first) { @@ -628,19 +577,6 @@ void ExpressionAnalyzer::normalizeTree() } -void ExpressionAnalyzer::addAliasColumns() -{ - if (!select_query) - return; - - if (!storage) - return; - - const auto & storage_aliases = storage->getColumns().aliases; - source_columns.insert(std::end(source_columns), std::begin(storage_aliases), std::end(storage_aliases)); -} - - void ExpressionAnalyzer::executeScalarSubqueries() { LogAST log; @@ -1285,19 +1221,24 @@ const ExpressionAnalyzer::AnalyzedJoin::JoinedColumnsList & ExpressionAnalyzer:: if (const ASTTablesInSelectQueryElement * node = select_query_with_join->join()) { const auto & table_expression = static_cast(*node->table_expression); - auto table_name_with_alias = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias table_name_with_alias(table_expression, context.getCurrentDatabase()); auto columns = getNamesAndTypeListFromTableExpression(table_expression, context); for (auto & column : columns) { - columns_from_joined_table.emplace_back(column, column.name); + JoinedColumn joined_column(column, column.name); if (source_columns.contains(column.name)) { auto qualified_name = table_name_with_alias.getQualifiedNamePrefix() + column.name; - columns_from_joined_table.back().name_and_type.name = qualified_name; + joined_column.name_and_type.name = qualified_name; } + + /// We don't want to select duplicate columns from the joined subquery if they appear + if (std::find(columns_from_joined_table.begin(), columns_from_joined_table.end(), joined_column) == columns_from_joined_table.end()) + columns_from_joined_table.push_back(joined_column); + } } } @@ -1343,8 +1284,8 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty if (table_to_join.database_and_table_name) { const auto & identifier = static_cast(*table_to_join.database_and_table_name); - auto database_table = getDatabaseAndTableNameFromIdentifier(identifier); - StoragePtr table = context.tryGetTable(database_table.first, database_table.second); + DatabaseAndTableWithAlias database_table(identifier); + StoragePtr table = context.tryGetTable(database_table.database, database_table.table); if (table) { @@ -1886,8 +1827,8 @@ void ExpressionAnalyzer::collectJoinedColumnsFromJoinOnExpr() const auto & left_table_expression = static_cast(*left_tables_element->table_expression); const auto & right_table_expression = static_cast(*right_tables_element->table_expression); - auto left_source_names = getTableNameWithAliasFromTableExpression(left_table_expression, context.getCurrentDatabase()); - auto right_source_names = getTableNameWithAliasFromTableExpression(right_table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias left_source_names(left_table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias right_source_names(right_table_expression, context.getCurrentDatabase()); /// Stores examples of columns which are only from one table. struct TableBelonging @@ -2040,7 +1981,7 @@ void ExpressionAnalyzer::collectJoinedColumns(NameSet & joined_columns) const auto & table_join = static_cast(*node->table_join); const auto & table_expression = static_cast(*node->table_expression); - auto joined_table_name = getTableNameWithAliasFromTableExpression(table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias joined_table_name(table_expression, context.getCurrentDatabase()); auto add_name_to_join_keys = [&](Names & join_keys, ASTs & join_asts, const ASTPtr & ast, bool right_table) { diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index 4117d8d3fe1..4f0ff267eae 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -259,6 +259,11 @@ private: JoinedColumn(const NameAndTypePair & name_and_type_, const String & original_name_) : name_and_type(name_and_type_), original_name(original_name_) {} + + bool operator==(const JoinedColumn & o) const + { + return name_and_type == o.name_and_type && original_name == o.original_name; + } }; using JoinedColumnsList = std::list; @@ -322,9 +327,6 @@ private: void optimizeIfWithConstantConditionImpl(ASTPtr & current_ast); bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & value) const; - /// Adds a list of ALIAS columns from the table. - void addAliasColumns(); - /// Replacing scalar subqueries with constant values. void executeScalarSubqueries(); diff --git a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h index f645059a03c..2e5cb910786 100644 --- a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h @@ -139,7 +139,7 @@ private: * instead of doing a subquery, you just need to read it. */ - auto database_and_table_name = ASTIdentifier::createSpecial(external_table_name); + auto database_and_table_name = createDatabaseAndTableNode("", external_table_name); if (auto ast_table_expr = typeid_cast(subquery_or_table_name_or_table_expression.get())) { diff --git a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 45105594c4f..58e28f9bfc6 100644 --- a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -81,40 +82,13 @@ void forEachTable(IAST * node, F && f) StoragePtr tryGetTable(const ASTPtr & database_and_table, const Context & context) { - String database; - String table; + const ASTIdentifier * id = typeid_cast(database_and_table.get()); + if (!id) + throw Exception("Logical error: identifier expected", ErrorCodes::LOGICAL_ERROR); - const ASTIdentifier * id = static_cast(database_and_table.get()); + DatabaseAndTableWithAlias db_and_table(*id); - if (id->children.empty()) - table = id->name; - else if (id->children.size() == 2) - { - database = static_cast(id->children[0].get())->name; - table = static_cast(id->children[1].get())->name; - } - else - throw Exception("Logical error: unexpected number of components in table expression", ErrorCodes::LOGICAL_ERROR); - - return context.tryGetTable(database, table); -} - - -void replaceDatabaseAndTable(ASTPtr & database_and_table, const String & database_name, const String & table_name) -{ - ASTPtr table = ASTIdentifier::createSpecial(table_name); - - if (!database_name.empty()) - { - ASTPtr database = ASTIdentifier::createSpecial(database_name); - - database_and_table = ASTIdentifier::createSpecial(database_name + "." + table_name); - database_and_table->children = {database, table}; - } - else - { - database_and_table = ASTIdentifier::createSpecial(table_name); - } + return context.tryGetTable(db_and_table.database, db_and_table.table); } } @@ -156,7 +130,7 @@ void InJoinSubqueriesPreprocessor::process(ASTSelectQuery * query) const forEachNonGlobalSubquery(query, [&] (IAST * subquery, IAST * function, IAST * table_join) { - forEachTable(subquery, [&] (ASTPtr & database_and_table) + forEachTable(subquery, [&] (ASTPtr & database_and_table) { StoragePtr storage = tryGetTable(database_and_table, context); @@ -199,7 +173,8 @@ void InJoinSubqueriesPreprocessor::process(ASTSelectQuery * query) const std::string table; std::tie(database, table) = getRemoteDatabaseAndTableName(*storage); - replaceDatabaseAndTable(database_and_table, database, table); + /// TODO: find a way to avoid AST node replacing + database_and_table = createDatabaseAndTableNode(database, table); } else throw Exception("InJoinSubqueriesPreprocessor: unexpected value of 'distributed_product_mode' setting", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 1d20f163925..31eedff6d11 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -27,7 +27,7 @@ BlockIO InterpreterAlterQuery::execute() auto & alter = typeid_cast(*query_ptr); if (!alter.cluster.empty()) - return executeDDLQueryOnCluster(query_ptr, context, {alter.table}); + return executeDDLQueryOnCluster(query_ptr, context, {alter.database}); const String & table_name = alter.table; String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index c1c4c6f0706..dc7225fbbaa 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -487,7 +487,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!create.to_table.empty()) databases.emplace(create.to_database); - return executeDDLQueryOnCluster(query_ptr, context, databases); + return executeDDLQueryOnCluster(query_ptr, context, std::move(databases)); } String path = context.getPath(); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index d303154c3a3..f058550a441 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 52bacb970c0..abeb7ff5c1f 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -47,7 +47,7 @@ BlockIO InterpreterRenameQuery::execute() databases.emplace(elem.to.database); } - return executeDDLQueryOnCluster(query_ptr, context, databases); + return executeDDLQueryOnCluster(query_ptr, context, std::move(databases)); } String path = context.getPath(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index ddac4a07611..d81c3b42bc0 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #include #include #include +#include namespace DB @@ -145,7 +147,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( max_streams = settings.max_threads; - const auto & table_expression = query.table(); + ASTPtr table_expression = getTableFunctionOrSubquery(query, 0); if (input) { @@ -204,7 +206,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (query_analyzer->isRewriteSubqueriesPredicate()) { /// remake interpreter_subquery when PredicateOptimizer is rewrite subqueries and main table is subquery - if (typeid_cast(table_expression.get())) + if (table_expression && typeid_cast(table_expression.get())) interpreter_subquery = std::make_unique( table_expression, getSubqueryContext(context), required_columns, QueryProcessingStage::Complete, subquery_depth + 1, only_analyze); @@ -235,29 +237,20 @@ InterpreterSelectQuery::InterpreterSelectQuery( void InterpreterSelectQuery::getDatabaseAndTableNames(String & database_name, String & table_name) { - auto query_database = query.database(); - auto query_table = query.table(); + if (auto db_and_table = getDatabaseAndTable(query, 0)) + { + table_name = db_and_table->table; + database_name = db_and_table->database; - /** If the table is not specified - use the table `system.one`. - * If the database is not specified - use the current database. - */ - if (query_database) - database_name = typeid_cast(*query_database).name; - if (query_table) - table_name = typeid_cast(*query_table).name; - - if (!query_table) + /// If the database is not specified - use the current database. + if (database_name.empty() && !context.tryGetTable("", table_name)) + database_name = context.getCurrentDatabase(); + } + else /// If the table is not specified - use the table `system.one`. { database_name = "system"; table_name = "one"; } - else if (!query_database) - { - if (context.tryGetTable("", table_name)) - database_name = ""; - else - database_name = context.getCurrentDatabase(); - } } @@ -883,8 +876,12 @@ void InterpreterSelectQuery::executeFetchColumns( /// If we need less number of columns that subquery have - update the interpreter. if (required_columns.size() < source_header.columns()) { + ASTPtr subquery = getTableFunctionOrSubquery(query, 0); + if (!subquery) + throw Exception("Subquery expected", ErrorCodes::LOGICAL_ERROR); + interpreter_subquery = std::make_unique( - query.table(), getSubqueryContext(context), required_columns, QueryProcessingStage::Complete, subquery_depth + 1, only_analyze); + subquery, getSubqueryContext(context), required_columns, QueryProcessingStage::Complete, subquery_depth + 1, only_analyze); if (query_analyzer->hasAggregation()) interpreter_subquery->ignoreWithTotals(); @@ -1293,6 +1290,17 @@ void InterpreterSelectQuery::executeUnion(Pipeline & pipeline) /// If there are still several streams, then we combine them into one if (pipeline.hasMoreThanOneStream()) { + /// Unify streams in case they have different headers. + auto first_header = pipeline.streams.at(0)->getHeader(); + for (size_t i = 1; i < pipeline.streams.size(); ++i) + { + auto & stream = pipeline.streams[i]; + auto header = stream->getHeader(); + auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name; + if (!blocksHaveEqualStructure(first_header, header)) + stream = std::make_shared(context, stream, first_header, mode); + } + pipeline.firstStream() = std::make_shared>(pipeline.streams, pipeline.stream_with_non_joined_data, max_streams); pipeline.stream_with_non_joined_data = nullptr; pipeline.streams.resize(1); @@ -1350,11 +1358,9 @@ bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) * In other cases, totals will be computed on the initiating server of the query, and it is not necessary to read the data to the end. */ - auto query_table = query.table(); - if (query_table) + if (auto query_table = getTableFunctionOrSubquery(query, 0)) { - auto ast_union = typeid_cast(query_table.get()); - if (ast_union) + if (auto ast_union = typeid_cast(query_table.get())) { for (const auto & elem : ast_union->list_of_selects->children) if (hasWithTotalsInAnySubqueryInFromClause(typeid_cast(*elem))) diff --git a/dbms/src/Interpreters/InterpreterShowProcesslistQuery.cpp b/dbms/src/Interpreters/InterpreterShowProcesslistQuery.cpp index 0c977a53ce8..697b286fe75 100644 --- a/dbms/src/Interpreters/InterpreterShowProcesslistQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowProcesslistQuery.cpp @@ -5,7 +5,6 @@ #include #include -#include namespace DB diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index df4332c5b16..ab15d1f0112 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index 1e4bb996f90..23fade42196 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -44,11 +44,8 @@ bool PredicateExpressionsOptimizer::optimizeImpl( /// split predicate with `and` PredicateExpressions outer_predicate_expressions = splitConjunctionPredicate(outer_expression); - std::vector tables_expression = getSelectTablesExpression(ast_select); - std::vector database_and_table_with_aliases; - for (const auto & table_expression : tables_expression) - database_and_table_with_aliases.emplace_back( - getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase())); + std::vector database_and_table_with_aliases = + getDatabaseAndTables(*ast_select, context.getCurrentDatabase()); bool is_rewrite_subquery = false; for (const auto & outer_predicate : outer_predicate_expressions) @@ -261,15 +258,14 @@ bool PredicateExpressionsOptimizer::optimizeExpression(const ASTPtr & outer_expr void PredicateExpressionsOptimizer::getAllSubqueryProjectionColumns(SubqueriesProjectionColumns & all_subquery_projection_columns) { - const auto tables_expression = getSelectTablesExpression(ast_select); + const auto tables_expression = getSelectTablesExpression(*ast_select); for (const auto & table_expression : tables_expression) { if (table_expression->subquery) { /// Use qualifiers to translate the columns of subqueries - const auto database_and_table_with_alias = - getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias database_and_table_with_alias(*table_expression, context.getCurrentDatabase()); String qualified_name_prefix = database_and_table_with_alias.getQualifiedNamePrefix(); getSubqueryProjectionColumns(all_subquery_projection_columns, qualified_name_prefix, static_cast(table_expression->subquery.get())->children[0]); @@ -336,7 +332,7 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que if (!select_query->tables || select_query->tables->children.empty()) return {}; - std::vector tables_expression = getSelectTablesExpression(select_query); + std::vector tables_expression = getSelectTablesExpression(*select_query); if (const auto qualified_asterisk = typeid_cast(asterisk.get())) { @@ -354,8 +350,7 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que for (auto it = tables_expression.begin(); it != tables_expression.end(); ++it) { const ASTTableExpression * table_expression = *it; - const auto database_and_table_with_alias = - getTableNameWithAliasFromTableExpression(*table_expression, context.getCurrentDatabase()); + DatabaseAndTableWithAlias database_and_table_with_alias(*table_expression, context.getCurrentDatabase()); /// database.table.* if (num_components == 2 && !database_and_table_with_alias.database.empty() && static_cast(*ident->children[0]).name == database_and_table_with_alias.database @@ -394,8 +389,8 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que else if (table_expression->database_and_table_name) { const auto database_and_table_ast = static_cast(table_expression->database_and_table_name.get()); - const auto database_and_table_name = getDatabaseAndTableNameFromIdentifier(*database_and_table_ast); - storage = context.getTable(database_and_table_name.first, database_and_table_name.second); + DatabaseAndTableWithAlias database_and_table_name(*database_and_table_ast); + storage = context.getTable(database_and_table_name.database, database_and_table_name.table); } const auto block = storage->getSampleBlock(); @@ -406,25 +401,6 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que return projection_columns; } -std::vector PredicateExpressionsOptimizer::getSelectTablesExpression(ASTSelectQuery * select_query) -{ - if (!select_query->tables) - return {}; - - std::vector tables_expression; - const ASTTablesInSelectQuery & tables_in_select_query = static_cast(*select_query->tables); - - for (const auto & child : tables_in_select_query.children) - { - ASTTablesInSelectQueryElement * tables_element = static_cast(child.get()); - - if (tables_element->table_expression) - tables_expression.emplace_back(static_cast(tables_element->table_expression.get())); - } - - return tables_expression; -} - void PredicateExpressionsOptimizer::cleanExpressionAlias(ASTPtr & expression) { const auto my_alias = expression->tryGetAlias(); diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h index c1a02a7df18..e999489475c 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.h +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace DB { @@ -105,8 +105,6 @@ private: ASTs getSelectQueryProjectionColumns(ASTPtr & ast); - std::vector getSelectTablesExpression(ASTSelectQuery * select_query); - ASTs evaluateAsterisk(ASTSelectQuery * select_query, const ASTPtr & asterisk); void cleanExpressionAlias(ASTPtr & expression); diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 557a006663d..3c31841d549 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -1,10 +1,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -51,28 +51,14 @@ static bool isUnlimitedQuery(const IAST * ast) if (!ast_selects->list_of_selects || ast_selects->list_of_selects->children.empty()) return false; - auto ast_select = typeid_cast(ast_selects->list_of_selects->children[0].get()); - + auto ast_select = typeid_cast(ast_selects->list_of_selects->children[0].get()); if (!ast_select) return false; - auto ast_database = ast_select->database(); - if (!ast_database) - return false; + if (auto database_and_table = getDatabaseAndTable(*ast_select, 0)) + return database_and_table->database == "system" && database_and_table->table == "processes"; - auto ast_table = ast_select->table(); - if (!ast_table) - return false; - - auto ast_database_id = typeid_cast(ast_database.get()); - if (!ast_database_id) - return false; - - auto ast_table_id = typeid_cast(ast_table.get()); - if (!ast_table_id) - return false; - - return ast_database_id->name == "system" && ast_table_id->name == "processes"; + return false; } return false; diff --git a/dbms/src/Interpreters/QueryNormalizer.cpp b/dbms/src/Interpreters/QueryNormalizer.cpp index 4aa2db7ad7f..eff50bfb235 100644 --- a/dbms/src/Interpreters/QueryNormalizer.cpp +++ b/dbms/src/Interpreters/QueryNormalizer.cpp @@ -9,7 +9,6 @@ #include #include #include -//#include #include namespace DB diff --git a/dbms/src/Interpreters/QueryNormalizer.h b/dbms/src/Interpreters/QueryNormalizer.h index fd0d8603b3e..376b3ba6e07 100644 --- a/dbms/src/Interpreters/QueryNormalizer.h +++ b/dbms/src/Interpreters/QueryNormalizer.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB { diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 6eb85e9c4df..13c800ed296 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -49,6 +49,7 @@ struct Settings M(SettingMilliseconds, connect_timeout_with_failover_ms, DBMS_DEFAULT_CONNECT_TIMEOUT_WITH_FAILOVER_MS, "Connection timeout for selecting first healthy replica.") \ M(SettingSeconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "") \ M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \ + M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \ M(SettingMilliseconds, queue_max_wait_ms, 5000, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \ diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 5ee99318c68..ebd575d314d 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -65,18 +65,23 @@ void TranslateQualifiedNamesVisitor::visit(ASTQualifiedAsterisk *, ASTPtr & ast, if (num_components > 2) throw Exception("Qualified asterisk cannot have more than two qualifiers", ErrorCodes::UNKNOWN_ELEMENT_IN_AST); + DatabaseAndTableWithAlias db_and_table(*ident); + for (const auto & table_names : tables) { /// database.table.*, table.* or alias.* - if ((num_components == 2 - && !table_names.database.empty() - && static_cast(*ident->children[0]).name == table_names.database - && static_cast(*ident->children[1]).name == table_names.table) - || (num_components == 0 - && ((!table_names.table.empty() && ident->name == table_names.table) - || (!table_names.alias.empty() && ident->name == table_names.alias)))) + if (num_components == 2) { - return; + if (!table_names.database.empty() && + db_and_table.database == table_names.database && + db_and_table.table == table_names.table) + return; + } + else if (num_components == 0) + { + if ((!table_names.table.empty() && db_and_table.table == table_names.table) || + (!table_names.alias.empty() && db_and_table.table == table_names.alias)) + return; } } diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h index 607e3de7217..eaf5a4f7ba0 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -5,7 +5,7 @@ #include #include -#include +#include namespace DB { @@ -18,7 +18,7 @@ struct ASTTableJoin; class NamesAndTypesList; -/// It visits nodes, find identifiers and translate their names to needed form. +/// It visits nodes, find columns (general identifiers and asterisks) and translate their names according to tables' names. class TranslateQualifiedNamesVisitor { public: diff --git a/dbms/src/Interpreters/evaluateQualified.cpp b/dbms/src/Interpreters/evaluateQualified.cpp deleted file mode 100644 index 205885011d1..00000000000 --- a/dbms/src/Interpreters/evaluateQualified.cpp +++ /dev/null @@ -1,167 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -namespace DB -{ - -/// Checks that ast is ASTIdentifier and remove num_qualifiers_to_strip components from left. -/// Example: 'database.table.name' -> (num_qualifiers_to_strip = 2) -> 'name'. -void stripIdentifier(DB::ASTPtr & ast, size_t num_qualifiers_to_strip) -{ - ASTIdentifier * identifier = typeid_cast(ast.get()); - - if (!identifier) - throw DB::Exception("ASTIdentifier expected for stripIdentifier", DB::ErrorCodes::LOGICAL_ERROR); - - if (num_qualifiers_to_strip) - { - size_t num_components = identifier->children.size(); - - /// plain column - if (num_components - num_qualifiers_to_strip == 1) - { - DB::String node_alias = identifier->tryGetAlias(); - ast = identifier->children.back(); - if (!node_alias.empty()) - ast->setAlias(node_alias); - } - else - /// nested column - { - identifier->children.erase(identifier->children.begin(), identifier->children.begin() + num_qualifiers_to_strip); - DB::String new_name; - for (const auto & child : identifier->children) - { - if (!new_name.empty()) - new_name += '.'; - new_name += static_cast(*child.get()).name; - } - identifier->name = new_name; - } - } -} - - -DatabaseAndTableWithAlias getTableNameWithAliasFromTableExpression(const ASTTableExpression & table_expression, - const String & current_database) -{ - DatabaseAndTableWithAlias database_and_table_with_alias; - - if (table_expression.database_and_table_name) - { - const auto & identifier = static_cast(*table_expression.database_and_table_name); - - database_and_table_with_alias.alias = identifier.tryGetAlias(); - - if (table_expression.database_and_table_name->children.empty()) - { - database_and_table_with_alias.database = current_database; - database_and_table_with_alias.table = identifier.name; - } - else - { - if (table_expression.database_and_table_name->children.size() != 2) - throw Exception("Logical error: number of components in table expression not equal to two", ErrorCodes::LOGICAL_ERROR); - - database_and_table_with_alias.database = static_cast(*identifier.children[0]).name; - database_and_table_with_alias.table = static_cast(*identifier.children[1]).name; - } - } - else if (table_expression.table_function) - { - database_and_table_with_alias.alias = table_expression.table_function->tryGetAlias(); - } - else if (table_expression.subquery) - { - database_and_table_with_alias.alias = table_expression.subquery->tryGetAlias(); - } - else - throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR); - - return database_and_table_with_alias; -} - -/// Get the number of components of identifier which are correspond to 'alias.', 'table.' or 'databas.table.' from names. -size_t getNumComponentsToStripInOrderToTranslateQualifiedName(const ASTIdentifier & identifier, - const DatabaseAndTableWithAlias & names) -{ - size_t num_qualifiers_to_strip = 0; - - auto get_identifier_name = [](const ASTPtr & ast) { return static_cast(*ast).name; }; - - /// It is compound identifier - if (!identifier.children.empty()) - { - size_t num_components = identifier.children.size(); - - /// database.table.column - if (num_components >= 3 - && !names.database.empty() - && get_identifier_name(identifier.children[0]) == names.database - && get_identifier_name(identifier.children[1]) == names.table) - { - num_qualifiers_to_strip = 2; - } - - /// table.column or alias.column. If num_components > 2, it is like table.nested.column. - if (num_components >= 2 - && ((!names.table.empty() && get_identifier_name(identifier.children[0]) == names.table) - || (!names.alias.empty() && get_identifier_name(identifier.children[0]) == names.alias))) - { - num_qualifiers_to_strip = 1; - } - } - - return num_qualifiers_to_strip; -} - -std::pair getDatabaseAndTableNameFromIdentifier(const ASTIdentifier & identifier) -{ - std::pair res; - res.second = identifier.name; - if (!identifier.children.empty()) - { - if (identifier.children.size() != 2) - throw Exception("Qualified table name could have only two components", ErrorCodes::LOGICAL_ERROR); - - res.first = typeid_cast(*identifier.children[0]).name; - res.second = typeid_cast(*identifier.children[1]).name; - } - return res; -} - -String DatabaseAndTableWithAlias::getQualifiedNamePrefix() const -{ - if (alias.empty() && table.empty()) - return ""; - - return (!alias.empty() ? alias : (database + '.' + table)) + '.'; -} - -void DatabaseAndTableWithAlias::makeQualifiedName(const ASTPtr & ast) const -{ - if (auto identifier = typeid_cast(ast.get())) - { - String prefix = getQualifiedNamePrefix(); - identifier->name.insert(identifier->name.begin(), prefix.begin(), prefix.end()); - - Names qualifiers; - if (!alias.empty()) - qualifiers.push_back(alias); - else - { - qualifiers.push_back(database); - qualifiers.push_back(table); - } - - for (const auto & qualifier : qualifiers) - identifier->children.emplace_back(std::make_shared(qualifier)); - } -} - -} diff --git a/dbms/src/Interpreters/interpretSubquery.cpp b/dbms/src/Interpreters/interpretSubquery.cpp index 305e76f7d4a..a585f7edc42 100644 --- a/dbms/src/Interpreters/interpretSubquery.cpp +++ b/dbms/src/Interpreters/interpretSubquery.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include namespace DB { @@ -69,10 +69,10 @@ std::shared_ptr interpretSubquery( } else { - auto database_table = getDatabaseAndTableNameFromIdentifier(*table); - const auto & storage = context.getTable(database_table.first, database_table.second); + DatabaseAndTableWithAlias database_table(*table); + const auto & storage = context.getTable(database_table.database, database_table.table); columns = storage->getColumns().ordinary; - select_query->replaceDatabaseAndTable(database_table.first, database_table.second); + select_query->replaceDatabaseAndTable(database_table.database, database_table.table); } select_expression_list->children.reserve(columns.size()); diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index 7081b512247..8869e873c71 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -187,18 +187,6 @@ ASTPtr ASTAlterQuery::clone() const return res; } -ASTPtr ASTAlterQuery::getRewrittenASTWithoutOnCluster(const std::string & new_database) const -{ - auto query_ptr = clone(); - auto & query = static_cast(*query_ptr); - - query.cluster.clear(); - if (query.database.empty()) - query.database = new_database; - - return query_ptr; -} - void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index ced92bfb9bd..9bfcdae2427 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -122,7 +122,10 @@ public: ASTPtr clone() const override; - ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override + { + return removeOnCluster(clone(), new_database); + } protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 100aee8e4f0..bbc7e35bff6 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -112,14 +112,7 @@ public: ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override { - auto query_ptr = clone(); - ASTCreateQuery & query = static_cast(*query_ptr); - - query.cluster.clear(); - if (query.database.empty()) - query.database = new_database; - - return query_ptr; + return removeOnCluster(clone(), new_database); } protected: diff --git a/dbms/src/Parsers/ASTDropQuery.cpp b/dbms/src/Parsers/ASTDropQuery.cpp index aa47829519a..6b6b9b0bec2 100644 --- a/dbms/src/Parsers/ASTDropQuery.cpp +++ b/dbms/src/Parsers/ASTDropQuery.cpp @@ -29,18 +29,6 @@ ASTPtr ASTDropQuery::clone() const return res; } -ASTPtr ASTDropQuery::getRewrittenASTWithoutOnCluster(const std::string & new_database) const -{ - auto query_ptr = clone(); - auto & query = static_cast(*query_ptr); - - query.cluster.clear(); - if (query.database.empty()) - query.database = new_database; - - return query_ptr; -} - void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { settings.ostr << (settings.hilite ? hilite_keyword : ""); diff --git a/dbms/src/Parsers/ASTDropQuery.h b/dbms/src/Parsers/ASTDropQuery.h index 0916eb757a0..83b5d28e38b 100644 --- a/dbms/src/Parsers/ASTDropQuery.h +++ b/dbms/src/Parsers/ASTDropQuery.h @@ -26,7 +26,10 @@ public: String getID() const override; ASTPtr clone() const override; - ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override + { + return removeOnCluster(clone(), new_database); + } protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/dbms/src/Parsers/ASTEnumElement.h b/dbms/src/Parsers/ASTEnumElement.h index 10b4e1e7482..cd07db53d04 100644 --- a/dbms/src/Parsers/ASTEnumElement.h +++ b/dbms/src/Parsers/ASTEnumElement.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB diff --git a/dbms/src/Parsers/ASTKillQueryQuery.cpp b/dbms/src/Parsers/ASTKillQueryQuery.cpp index 0f3e5406fdd..a8b351cdb39 100644 --- a/dbms/src/Parsers/ASTKillQueryQuery.cpp +++ b/dbms/src/Parsers/ASTKillQueryQuery.cpp @@ -8,16 +8,6 @@ String ASTKillQueryQuery::getID() const return "KillQueryQuery_" + (where_expression ? where_expression->getID() : "") + "_" + String(sync ? "SYNC" : "ASYNC"); } -ASTPtr ASTKillQueryQuery::getRewrittenASTWithoutOnCluster(const std::string & /*new_database*/) const -{ - auto query_ptr = clone(); - ASTKillQueryQuery & query = static_cast(*query_ptr); - - query.cluster.clear(); - - return query_ptr; -} - void ASTKillQueryQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { settings.ostr << (settings.hilite ? hilite_keyword : "") << "KILL QUERY "; diff --git a/dbms/src/Parsers/ASTKillQueryQuery.h b/dbms/src/Parsers/ASTKillQueryQuery.h index 0592062ccd6..491bd3aecd2 100644 --- a/dbms/src/Parsers/ASTKillQueryQuery.h +++ b/dbms/src/Parsers/ASTKillQueryQuery.h @@ -24,7 +24,10 @@ public: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - ASTPtr getRewrittenASTWithoutOnCluster(const std::string &new_database) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override + { + return removeOnCluster(clone()); + } }; } diff --git a/dbms/src/Parsers/ASTOptimizeQuery.cpp b/dbms/src/Parsers/ASTOptimizeQuery.cpp index dd37b665173..5e95dc41795 100644 --- a/dbms/src/Parsers/ASTOptimizeQuery.cpp +++ b/dbms/src/Parsers/ASTOptimizeQuery.cpp @@ -3,19 +3,6 @@ namespace DB { - -ASTPtr ASTOptimizeQuery::getRewrittenASTWithoutOnCluster(const std::string & new_database) const -{ - auto query_ptr = clone(); - ASTOptimizeQuery & query = static_cast(*query_ptr); - - query.cluster.clear(); - if (query.database.empty()) - query.database = new_database; - - return query_ptr; -} - void ASTOptimizeQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { settings.ostr << (settings.hilite ? hilite_keyword : "") << "OPTIMIZE TABLE " << (settings.hilite ? hilite_none : "") diff --git a/dbms/src/Parsers/ASTOptimizeQuery.h b/dbms/src/Parsers/ASTOptimizeQuery.h index 269ea5b19ff..d228a8c905f 100644 --- a/dbms/src/Parsers/ASTOptimizeQuery.h +++ b/dbms/src/Parsers/ASTOptimizeQuery.h @@ -40,7 +40,10 @@ public: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - ASTPtr getRewrittenASTWithoutOnCluster(const std::string &new_database) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &new_database) const override + { + return removeOnCluster(clone(), new_database); + } }; } diff --git a/dbms/src/Parsers/ASTQueryWithOnCluster.cpp b/dbms/src/Parsers/ASTQueryWithOnCluster.cpp index f37b741081e..9519a33c1e5 100644 --- a/dbms/src/Parsers/ASTQueryWithOnCluster.cpp +++ b/dbms/src/Parsers/ASTQueryWithOnCluster.cpp @@ -2,8 +2,6 @@ #include #include #include -#include -#include #include #include #include diff --git a/dbms/src/Parsers/ASTQueryWithOnCluster.h b/dbms/src/Parsers/ASTQueryWithOnCluster.h index f850dd9014c..373307700aa 100644 --- a/dbms/src/Parsers/ASTQueryWithOnCluster.h +++ b/dbms/src/Parsers/ASTQueryWithOnCluster.h @@ -28,6 +28,27 @@ public: static bool parse(Pos & pos, std::string & cluster_str, Expected & expected); virtual ~ASTQueryWithOnCluster() = default; + +protected: + template + static ASTPtr removeOnCluster(ASTPtr query_ptr, const std::string & new_database) + { + T & query = static_cast(*query_ptr); + + query.cluster.clear(); + if (query.database.empty()) + query.database = new_database; + + return query_ptr; + } + + template + static ASTPtr removeOnCluster(ASTPtr query_ptr) + { + T & query = static_cast(*query_ptr); + query.cluster.clear(); + return query_ptr; + } }; } diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp index 0cf19a28851..efbbe6066bd 100644 --- a/dbms/src/Parsers/ASTSelectQuery.cpp +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -19,6 +19,19 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +ASTPtr createDatabaseAndTableNode(const String & database_name, const String & table_name) +{ + if (database_name.empty()) + return ASTIdentifier::createSpecial(table_name); + + ASTPtr database = ASTIdentifier::createSpecial(database_name); + ASTPtr table = ASTIdentifier::createSpecial(table_name); + + ASTPtr database_and_table = ASTIdentifier::createSpecial(database_name + "." + table_name); + database_and_table->children = {database, table}; + return database_and_table; +} + ASTPtr ASTSelectQuery::clone() const { @@ -242,46 +255,6 @@ static const ASTTablesInSelectQueryElement * getFirstTableJoin(const ASTSelectQu } -ASTPtr ASTSelectQuery::database() const -{ - const ASTTableExpression * table_expression = getFirstTableExpression(*this); - if (!table_expression || !table_expression->database_and_table_name || table_expression->database_and_table_name->children.empty()) - return {}; - - if (table_expression->database_and_table_name->children.size() != 2) - throw Exception("Logical error: more than two components in table expression", ErrorCodes::LOGICAL_ERROR); - - return table_expression->database_and_table_name->children[0]; -} - - -ASTPtr ASTSelectQuery::table() const -{ - const ASTTableExpression * table_expression = getFirstTableExpression(*this); - if (!table_expression) - return {}; - - if (table_expression->database_and_table_name) - { - if (table_expression->database_and_table_name->children.empty()) - return table_expression->database_and_table_name; - - if (table_expression->database_and_table_name->children.size() != 2) - throw Exception("Logical error: more than two components in table expression", ErrorCodes::LOGICAL_ERROR); - - return table_expression->database_and_table_name->children[1]; - } - - if (table_expression->table_function) - return table_expression->table_function; - - if (table_expression->subquery) - return static_cast(table_expression->subquery.get())->children.at(0); - - throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); -} - - ASTPtr ASTSelectQuery::sample_size() const { const ASTTableExpression * table_expression = getFirstTableExpression(*this); @@ -363,12 +336,9 @@ void ASTSelectQuery::setDatabaseIfNeeded(const String & database_name) } else if (table_expression->database_and_table_name->children.empty()) { - ASTPtr database = ASTIdentifier::createSpecial(database_name); - ASTPtr table = table_expression->database_and_table_name; + const ASTIdentifier & identifier = static_cast(*table_expression->database_and_table_name); - const String & old_name = static_cast(*table_expression->database_and_table_name).name; - table_expression->database_and_table_name = ASTIdentifier::createSpecial(database_name + "." + old_name); - table_expression->database_and_table_name->children = {database, table}; + table_expression->database_and_table_name = createDatabaseAndTableNode(database_name, identifier.name); } else if (table_expression->database_and_table_name->children.size() != 2) { @@ -396,19 +366,7 @@ void ASTSelectQuery::replaceDatabaseAndTable(const String & database_name, const table_expression = table_expr.get(); } - ASTPtr table = ASTIdentifier::createSpecial(table_name); - - if (!database_name.empty()) - { - ASTPtr database = ASTIdentifier::createSpecial(database_name); - - table_expression->database_and_table_name = ASTIdentifier::createSpecial(database_name + "." + table_name); - table_expression->database_and_table_name->children = {database, table}; - } - else - { - table_expression->database_and_table_name = ASTIdentifier::createSpecial(table_name); - } + table_expression->database_and_table_name = createDatabaseAndTableNode(database_name, table_name); } diff --git a/dbms/src/Parsers/ASTSelectQuery.h b/dbms/src/Parsers/ASTSelectQuery.h index 0ffdb44395e..1c88186cb31 100644 --- a/dbms/src/Parsers/ASTSelectQuery.h +++ b/dbms/src/Parsers/ASTSelectQuery.h @@ -39,8 +39,6 @@ public: ASTPtr settings; /// Compatibility with old parser of tables list. TODO remove - ASTPtr database() const; - ASTPtr table() const; ASTPtr sample_size() const; ASTPtr sample_offset() const; ASTPtr array_join_expression_list() const; @@ -55,4 +53,7 @@ protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; + +ASTPtr createDatabaseAndTableNode(const String & database_name, const String & table_name); + } diff --git a/dbms/src/Parsers/ParserDescribeTableQuery.cpp b/dbms/src/Parsers/ParserDescribeTableQuery.cpp index ebfc3baa33f..0f768e22324 100644 --- a/dbms/src/Parsers/ParserDescribeTableQuery.cpp +++ b/dbms/src/Parsers/ParserDescribeTableQuery.cpp @@ -1,4 +1,3 @@ -#include #include #include diff --git a/dbms/src/Parsers/ParserInsertQuery.cpp b/dbms/src/Parsers/ParserInsertQuery.cpp index 399aa43ea98..73aca09c210 100644 --- a/dbms/src/Parsers/ParserInsertQuery.cpp +++ b/dbms/src/Parsers/ParserInsertQuery.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/dbms/src/Parsers/ParserOptimizeQuery.cpp b/dbms/src/Parsers/ParserOptimizeQuery.cpp index e0dcf7ffb47..835db12cbb3 100644 --- a/dbms/src/Parsers/ParserOptimizeQuery.cpp +++ b/dbms/src/Parsers/ParserOptimizeQuery.cpp @@ -4,7 +4,6 @@ #include #include -#include #include diff --git a/dbms/src/Parsers/ParserSelectQuery.cpp b/dbms/src/Parsers/ParserSelectQuery.cpp index ffd9273dd8a..74f2e735ccd 100644 --- a/dbms/src/Parsers/ParserSelectQuery.cpp +++ b/dbms/src/Parsers/ParserSelectQuery.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Parsers/ParserSystemQuery.cpp b/dbms/src/Parsers/ParserSystemQuery.cpp index 87140a3f6f7..1bf7c7219dc 100644 --- a/dbms/src/Parsers/ParserSystemQuery.cpp +++ b/dbms/src/Parsers/ParserSystemQuery.cpp @@ -1,8 +1,6 @@ #include #include #include -#include -#include #include #include #include diff --git a/dbms/src/Parsers/ParserTablesInSelectQuery.cpp b/dbms/src/Parsers/ParserTablesInSelectQuery.cpp index 088cd567fba..0ba2a403a94 100644 --- a/dbms/src/Parsers/ParserTablesInSelectQuery.cpp +++ b/dbms/src/Parsers/ParserTablesInSelectQuery.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index fb55db8bbb9..adaffed6c7c 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index 30a7ce1f733..745e1112a1f 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB diff --git a/dbms/src/Storages/MergeTree/KeyCondition.h b/dbms/src/Storages/MergeTree/KeyCondition.h index e2eb179dcba..cb06909eb87 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.h +++ b/dbms/src/Storages/MergeTree/KeyCondition.h @@ -10,7 +10,6 @@ #include #include #include -#include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index a31d12d932f..1c959f41759 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 6e2b2d3a20a..196259733c4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 33bdb8d2aaa..5b995e40853 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -10,6 +9,11 @@ namespace ProfileEvents extern const Event ReadBackoff; } +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace DB { @@ -22,8 +26,13 @@ MergeTreeReadPool::MergeTreeReadPool( const bool do_not_steal_tasks) : backoff_settings{backoff_settings}, backoff_state{threads}, data{data}, column_names{column_names}, do_not_steal_tasks{do_not_steal_tasks}, - predict_block_size_bytes{preferred_block_size_bytes > 0}, prewhere_info{prewhere_info} + predict_block_size_bytes{preferred_block_size_bytes > 0}, prewhere_info{prewhere_info}, parts_ranges{parts} { + /// reverse from right-to-left to left-to-right + /// because 'reverse' was done in MergeTreeDataSelectExecutor + for (auto & part_ranges : parts_ranges) + std::reverse(std::begin(part_ranges.ranges), std::end(part_ranges.ranges)); + /// parts don't contain duplicate MergeTreeDataPart's. const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_info, check_columns); fillPerThreadInfo(threads, sum_marks, per_part_sum_marks, parts, min_marks_for_concurrent_read); @@ -120,6 +129,27 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, prewhere_info && prewhere_info->remove_prewhere_column, per_part_should_reorder[part_idx], std::move(curr_task_size_predictor)); } +MarkRanges MergeTreeReadPool::getRestMarks(const std::string & part_path, const MarkRange & from) const +{ + MarkRanges all_part_ranges; + for (const auto & part_ranges : parts_ranges) + { + if (part_ranges.data_part->getFullPath() == part_path) + { + all_part_ranges = part_ranges.ranges; + break; + } + } + if (all_part_ranges.empty()) + throw Exception("Trying to read marks range [" + std::to_string(from.begin) + ", " + std::to_string(from.end) + "] from part '" + + part_path + "' which has no ranges in this query", ErrorCodes::LOGICAL_ERROR); + + auto begin = std::lower_bound(all_part_ranges.begin(), all_part_ranges.end(), from, [] (const auto & f, const auto & s) { return f.begin < s.begin; }); + if (begin == all_part_ranges.end()) + begin = std::prev(all_part_ranges.end()); + begin->begin = from.begin; + return MarkRanges(begin, all_part_ranges.end()); +} Block MergeTreeReadPool::getHeader() const { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h index c0d10d01be4..47f4258116f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h @@ -80,6 +80,9 @@ public: */ void profileFeedback(const ReadBufferFromFileBase::ProfileInfo info); + /// This method tells which mark ranges we have to read if we start from @from mark range + MarkRanges getRestMarks(const std::string & part_path, const MarkRange & from) const; + Block getHeader() const; private: @@ -127,6 +130,8 @@ private: std::set remaining_thread_tasks; + RangesInDataParts parts_ranges; + mutable std::mutex mutex; Logger * log = &Logger::get("MergeTreeReadPool"); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 9a4d9c0e9c2..f7fa9bf6703 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -149,7 +149,7 @@ struct MergeTreeSettings M(SettingUInt64, finished_mutations_to_keep, 100) \ \ /** Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled) */ \ - M(SettingUInt64, min_merge_bytes_to_use_direct_io, 0) + M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024) /// Settings that should not change after the creation of a table. #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \ diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp index b3e697b9b5a..3e4a96dc2e4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp @@ -65,50 +65,49 @@ bool MergeTreeThreadBlockInputStream::getNewTask() } const std::string path = task->data_part->getFullPath(); - size_t current_task_first_mark = task->mark_ranges[0].begin; - size_t current_task_end_mark = task->mark_ranges.back().end; /// Allows pool to reduce number of threads in case of too slow reads. auto profile_callback = [this](ReadBufferFromFileBase::ProfileInfo info) { pool->profileFeedback(info); }; if (!reader) { + auto rest_mark_ranges = pool->getRestMarks(path, task->mark_ranges[0]); + if (use_uncompressed_cache) owned_uncompressed_cache = storage.context.getUncompressedCache(); - owned_mark_cache = storage.context.getMarkCache(); reader = std::make_unique( path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); + storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); if (prewhere_info) pre_reader = std::make_unique( path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, task->mark_ranges, min_bytes_to_use_direct_io, + storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); } else { - /// in other case we can reuse readers, they stopped exactly at required position - if (last_task_end_mark != current_task_first_mark || path != last_readed_part_path) + /// in other case we can reuse readers, anyway they will be "seeked" to required mark + if (path != last_readed_part_path) { + auto rest_mark_ranges = pool->getRestMarks(path, task->mark_ranges[0]); /// retain avg_value_size_hints reader = std::make_unique( path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, + storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, reader->getAvgValueSizeHints(), profile_callback); if (prewhere_info) pre_reader = std::make_unique( path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, task->mark_ranges, min_bytes_to_use_direct_io, + storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, pre_reader->getAvgValueSizeHints(), profile_callback); } } last_readed_part_path = path; - last_task_end_mark = current_task_end_mark; return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h index be47ba3cac9..1a2009aac95 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h @@ -44,8 +44,6 @@ private: std::shared_ptr pool; size_t min_marks_to_read; - /// Last readed mark in task for this thread - size_t last_task_end_mark; /// Last part readed in this thread std::string last_readed_part_path; /// Names from header. Used in order to order columns in read blocks. diff --git a/dbms/src/Storages/MutationCommands.cpp b/dbms/src/Storages/MutationCommands.cpp index 685aa883190..6ba9c23a257 100644 --- a/dbms/src/Storages/MutationCommands.cpp +++ b/dbms/src/Storages/MutationCommands.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 7b442169de5..8144639d998 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 05a61343108..546e8f27843 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 022f77fa5e4..d89f818eaaa 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -2,11 +2,11 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -27,23 +27,25 @@ namespace ErrorCodes static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name) { - auto query_table = query.table(); + auto db_and_table = getDatabaseAndTable(query, 0); + ASTPtr subquery = getTableFunctionOrSubquery(query, 0); - if (!query_table) + if (!db_and_table && !subquery) return; - if (auto ast_id = typeid_cast(query_table.get())) + if (db_and_table) { - auto query_database = query.database(); + select_table_name = db_and_table->table; - if (!query_database) + if (db_and_table->database.empty()) + { + db_and_table->database = select_database_name; query.setDatabaseIfNeeded(select_database_name); - - select_table_name = ast_id->name; - select_database_name = query_database ? typeid_cast(*query_database).name : select_database_name; - + } + else + select_database_name = db_and_table->database; } - else if (auto ast_select = typeid_cast(query_table.get())) + else if (auto ast_select = typeid_cast(subquery.get())) { if (ast_select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); @@ -64,12 +66,11 @@ static void checkAllowedQueries(const ASTSelectQuery & query) if (query.prewhere_expression || query.final() || query.sample_size()) throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - auto query_table = query.table(); - - if (!query_table) + ASTPtr subquery = getTableFunctionOrSubquery(query, 0); + if (!subquery) return; - if (auto ast_select = typeid_cast(query_table.get())) + if (auto ast_select = typeid_cast(subquery.get())) { if (ast_select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index ebda49c15e9..0bb5c832cfe 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index 84091d83a05..8f17799a232 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include diff --git a/dbms/src/Storages/System/CMakeLists.txt b/dbms/src/Storages/System/CMakeLists.txt index 907fbc2907a..ed12cf6f78c 100644 --- a/dbms/src/Storages/System/CMakeLists.txt +++ b/dbms/src/Storages/System/CMakeLists.txt @@ -1,5 +1,16 @@ +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/StorageSystemContributors.generated.cpp) + execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/StorageSystemContributors.sh) +endif() + +set (CONFIG_BUILD ${CMAKE_CURRENT_BINARY_DIR}/StorageSystemBuildOptions.generated.cpp) +get_property (BUILD_COMPILE_DEFINITIONS DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY COMPILE_DEFINITIONS) +get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) +string (TIMESTAMP BUILD_DATE "%Y-%m-%d" UTC) +configure_file (StorageSystemBuildOptions.generated.cpp.in ${CONFIG_BUILD}) include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(storages_system .) +list (APPEND storages_system_sources ${CONFIG_BUILD}) add_library(clickhouse_storages_system ${LINK_MODE} ${storages_system_headers} ${storages_system_sources}) target_link_libraries(clickhouse_storages_system dbms) +target_include_directories(clickhouse_storages_system PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp index 2a8ffc947be..b85cc9cf9f7 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp @@ -1,7 +1,9 @@ -#include +#include "StorageSystemBuildOptions.h" + #include #include -#include + +extern const char * auto_config_build[]; namespace DB { diff --git a/dbms/src/Common/config_build.cpp.in b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in similarity index 98% rename from dbms/src/Common/config_build.cpp.in rename to dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 9e74ea43e00..57c418d7f98 100644 --- a/dbms/src/Common/config_build.cpp.in +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -1,7 +1,5 @@ // .cpp autogenerated by cmake -#include - const char * auto_config_build[] { "VERSION_FULL", "@VERSION_FULL@", diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 40802f16466..66f56996ddb 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include diff --git a/dbms/src/Storages/System/StorageSystemContributors.cpp b/dbms/src/Storages/System/StorageSystemContributors.cpp new file mode 100644 index 00000000000..99c720e0f97 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemContributors.cpp @@ -0,0 +1,36 @@ +#if __has_include("StorageSystemContributors.generated.cpp") + +#include "StorageSystemContributors.h" + +#include +#include +#include +#include + + +extern const char * auto_contributors[]; + +namespace DB +{ +NamesAndTypesList StorageSystemContributors::getNamesAndTypes() +{ + return { + {"name", std::make_shared()}, + }; +} + +void StorageSystemContributors::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const +{ + std::vector contributors; + for (auto it = auto_contributors; *it; ++it) + contributors.emplace_back(*it); + + pcg64 rng(randomSeed()); + std::shuffle(contributors.begin(), contributors.end(), rng); + + for (auto & it : contributors) + res_columns[0]->insert(String(it)); +} +} + +#endif diff --git a/dbms/src/Storages/System/StorageSystemContributors.h b/dbms/src/Storages/System/StorageSystemContributors.h new file mode 100644 index 00000000000..b62895d5788 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemContributors.h @@ -0,0 +1,34 @@ +#pragma once + +#if __has_include("StorageSystemContributors.generated.cpp") + +#include +#include + + +namespace DB +{ +class Context; + + +/** System table "contributors" with list of clickhouse contributors + */ +class StorageSystemContributors : public ext::shared_ptr_helper, + public IStorageSystemOneBlock +{ +protected: + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; + + using IStorageSystemOneBlock::IStorageSystemOneBlock; + +public: + std::string getName() const override + { + return "SystemContributors"; + } + + static NamesAndTypesList getNamesAndTypes(); +}; +} + +#endif diff --git a/dbms/src/Storages/System/StorageSystemContributors.sh b/dbms/src/Storages/System/StorageSystemContributors.sh new file mode 100755 index 00000000000..44b4730a6f4 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemContributors.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +CONTRIBUTORS_FILE=${CONTRIBUTORS_FILE=$CUR_DIR/StorageSystemContributors.generated.cpp} + +git shortlog --summary | perl -lnE 's/^\s+\d+\s+(.+)/"$1",/; next unless $1; say $_' > $CONTRIBUTORS_FILE.tmp + +# If git history not available - dont make target file +if [ ! -s $CONTRIBUTORS_FILE.tmp ]; then + exit +fi + +echo "// autogenerated by $0" > $CONTRIBUTORS_FILE +echo "const char * auto_contributors[] {" >> $CONTRIBUTORS_FILE +cat $CONTRIBUTORS_FILE.tmp >> $CONTRIBUTORS_FILE +echo "nullptr };" >> $CONTRIBUTORS_FILE + +rm $CONTRIBUTORS_FILE.tmp diff --git a/dbms/src/Storages/System/attachSystemTables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp index 10448948942..facaa6c4fd3 100644 --- a/dbms/src/Storages/System/attachSystemTables.cpp +++ b/dbms/src/Storages/System/attachSystemTables.cpp @@ -32,6 +32,9 @@ #include #include #include +#if __has_include("StorageSystemContributors.generated.cpp") +# include +#endif namespace DB @@ -56,6 +59,9 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("data_type_families", StorageSystemDataTypeFamilies::create("data_type_families")); system_database.attachTable("collations", StorageSystemCollations::create("collations")); system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines")); +#if __has_include("StorageSystemContributors.generated.cpp") + system_database.attachTable("contributors", StorageSystemContributors::create("contributors")); +#endif } void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index 2615e8d0d92..d143cb32ff8 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -119,16 +119,23 @@ String transformQueryForExternalDatabase( { if (function->name == "and") { + bool compatible_found = false; auto new_function_and = std::make_shared(); auto new_function_and_arguments = std::make_shared(); new_function_and->arguments = new_function_and_arguments; new_function_and->children.push_back(new_function_and_arguments); for (const auto & elem : function->arguments->children) + { if (isCompatible(*elem)) + { new_function_and_arguments->children.push_back(elem); + compatible_found = true; + } + } - select->where_expression = std::move(new_function_and); + if (compatible_found) + select->where_expression = std::move(new_function_and); } } } diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index 45aa42fea32..b5dace28d41 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include #include diff --git a/dbms/tests/CMakeLists.txt b/dbms/tests/CMakeLists.txt index 3f42f94158b..6a983134937 100644 --- a/dbms/tests/CMakeLists.txt +++ b/dbms/tests/CMakeLists.txt @@ -27,7 +27,7 @@ if (ENABLE_TESTS) # maybe add --no-long ? # if you want disable some tests: env TEST_OPT0='--skip compile' - add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} INTERNAL_COMPILER_BIN_ROOT=${INTERNAL_COMPILER_BIN_ROOT} ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server") + add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server") endif () if (ENABLE_TEST_INTEGRATION) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index bf71c18c5c7..b9f4c82af62 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -242,7 +242,7 @@ def main(args): stderr_element = et.Element("system-err") stderr_element.text = et.CDATA(stderr) report_testcase.append(stderr_element) - print(stderr) + print(stderr.encode('utf-8')) if args.stop and ('Connection refused' in stderr or 'Attempt to read after eof' in stderr) and not 'Received exception from server' in stderr: SERVER_DIED = True diff --git a/dbms/tests/external_dictionaries/generate_and_test.py b/dbms/tests/external_dictionaries/generate_and_test.py index 98b67345326..2774ccaa6d3 100755 --- a/dbms/tests/external_dictionaries/generate_and_test.py +++ b/dbms/tests/external_dictionaries/generate_and_test.py @@ -141,10 +141,17 @@ def generate_structure(args): base_name = 'range_hashed_' + range_hashed_range_type dictionaries.extend([ [ 'file_' + base_name, 3, False ], - # [ 'clickhouse_' + base_name, 3, True ], + [ 'clickhouse_' + base_name, 3, False ], # [ 'executable_flat' + base_name, 3, True ] ]) + if not args.no_mysql: + for range_hashed_range_type in range_hashed_range_types: + base_name = 'range_hashed_' + range_hashed_range_type + dictionaries.extend([ + ['mysql_' + base_name, 3, False], + ]) + files = [ 'key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv', 'key_range_hashed_{range_hashed_range_type}.tsv' ] @@ -206,6 +213,36 @@ range_hashed_dictGet_values = { ("toDateTime('2015-11-19 23:59:59')", "toDateTime('2015-10-26 00:00:01')", "toDateTime('2018-09-14 00:00:00')")], } +range_hashed_mysql_column_types = { + 'UInt8': 'tinyint unsigned', + 'UInt16': 'smallint unsigned', + 'UInt32': 'int unsigned', + 'UInt64': 'bigint unsigned', + 'Int8': 'tinyint', + 'Int16': 'smallint', + 'Int32': 'int', + 'Int64': 'bigint', + # default type (Date) for compatibility with older versions: + '': 'date', + 'Date': 'date', + 'DateTime': 'datetime', +} + +range_hashed_clickhouse_column_types = { + 'UInt8': 'UInt8', + 'UInt16': 'UInt16', + 'UInt32': 'UInt32', + 'UInt64': 'UInt64', + 'Int8': 'Int8', + 'Int16': 'Int16', + 'Int32': 'Int32', + 'Int64': 'Int64', + # default type (Date) for compatibility with older versions: + '': 'Date', + 'Date': 'Date', + 'DateTime': 'DateTime', +} + def dump_report(destination, suite, test_case, report): if destination is not None: @@ -268,6 +305,41 @@ def generate_data(args): query = file_source_query % comma_separated(chain(keys, columns(), ['Parent'] if 1 == len(keys) else [])) call([args.client, '--port', args.port, '--query', query], 'generated/' + file) + table_name = "test.dictionary_source_" + range_hashed_range_type + col_type = range_hashed_clickhouse_column_types[range_hashed_range_type] + + source_tsv_full_path = "{0}/generated/{1}".format(prefix, file) + print 'Creating Clickhouse table for "{0}" range_hashed dictionary...'.format(range_hashed_range_type) + system('cat {source} | {ch} --port={port} -m -n --query "' + 'create database if not exists test;' + 'drop table if exists {table_name};' + 'create table {table_name} (' + 'id UInt64, StartDate {col_type}, EndDate {col_type},' + 'UInt8_ UInt8, UInt16_ UInt16, UInt32_ UInt32, UInt64_ UInt64,' + 'Int8_ Int8, Int16_ Int16, Int32_ Int32, Int64_ Int64,' + 'Float32_ Float32, Float64_ Float64,' + 'String_ String,' + 'Date_ Date, DateTime_ DateTime, UUID_ UUID' + ') engine=Log; insert into {table_name} format TabSeparated' + '"'.format(table_name=table_name, col_type=col_type, source=source_tsv_full_path, ch=args.client, port=args.port)) + + if not args.no_mysql: + print 'Creating MySQL table for "{0}" range_hashed dictionary...'.format(range_hashed_range_type) + col_type = range_hashed_mysql_column_types[range_hashed_range_type] + subprocess.check_call('echo "' + 'create database if not exists test;' + 'drop table if exists {table_name};' + 'create table {table_name} (' + 'id tinyint unsigned, StartDate {col_type}, EndDate {col_type}, ' + 'UInt8_ tinyint unsigned, UInt16_ smallint unsigned, UInt32_ int unsigned, UInt64_ bigint unsigned, ' + 'Int8_ tinyint, Int16_ smallint, Int32_ int, Int64_ bigint, ' + 'Float32_ float, Float64_ double, ' + 'String_ text, Date_ date, DateTime_ datetime, UUID_ varchar(36)' + ');' + 'load data local infile \'{source}\' into table {table_name};" | mysql $MYSQL_OPTIONS --local-infile=1' + .format(prefix, table_name=table_name, col_type=col_type, source=source_tsv_full_path), shell=True) + + # create MySQL table from complete_query if not args.no_mysql: print 'Creating MySQL table' @@ -365,7 +437,7 @@ def generate_dictionaries(args): default test - dictionary_source
+ dictionary_source{key_type}
''' % args.port @@ -384,7 +456,7 @@ def generate_dictionaries(args): root test - dictionary_source
+ dictionary_source{key_type}
''' @@ -518,33 +590,34 @@ def generate_dictionaries(args): ''' + source_clickhouse_deafult = source_clickhouse.format(key_type="") sources_and_layouts = [ # Simple key dictionaries [ source_file % (generated_prefix + files[0]), layout_flat], - [ source_clickhouse, layout_flat ], + [ source_clickhouse_deafult, layout_flat ], [ source_executable % (generated_prefix + files[0]), layout_flat ], [ source_file % (generated_prefix + files[0]), layout_hashed], - [ source_clickhouse, layout_hashed ], + [ source_clickhouse_deafult, layout_hashed ], [ source_executable % (generated_prefix + files[0]), layout_hashed ], - [ source_clickhouse, layout_cache ], + [ source_clickhouse_deafult, layout_cache ], [ source_executable_cache % (generated_prefix + files[0]), layout_cache ], # Complex key dictionaries with (UInt8, UInt8) key [ source_file % (generated_prefix + files[1]), layout_complex_key_hashed], - [ source_clickhouse, layout_complex_key_hashed ], + [ source_clickhouse_deafult, layout_complex_key_hashed ], [ source_executable % (generated_prefix + files[1]), layout_complex_key_hashed ], - [ source_clickhouse, layout_complex_key_cache ], + [ source_clickhouse_deafult, layout_complex_key_cache ], [ source_executable_cache % (generated_prefix + files[1]), layout_complex_key_cache ], # Complex key dictionaries with (String, UInt8) key [ source_file % (generated_prefix + files[2]), layout_complex_key_hashed], - [ source_clickhouse, layout_complex_key_hashed ], + [ source_clickhouse_deafult, layout_complex_key_hashed ], [ source_executable % (generated_prefix + files[2]), layout_complex_key_hashed ], - [ source_clickhouse, layout_complex_key_cache ], + [ source_clickhouse_deafult, layout_complex_key_cache ], [ source_executable_cache % (generated_prefix + files[2]), layout_complex_key_cache ], ] @@ -568,14 +641,15 @@ def generate_dictionaries(args): ]) if not args.no_mysql: + source_mysql_default = source_mysql.format(key_type="") sources_and_layouts.extend([ - [ source_mysql, layout_flat ], - [ source_mysql, layout_hashed ], - [ source_mysql, layout_cache ], - [ source_mysql, layout_complex_key_hashed ], - [ source_mysql, layout_complex_key_cache ], - [ source_mysql, layout_complex_key_hashed ], - [ source_mysql, layout_complex_key_cache ], + [ source_mysql_default, layout_flat ], + [ source_mysql_default, layout_hashed ], + [ source_mysql_default, layout_cache ], + [ source_mysql_default, layout_complex_key_hashed ], + [ source_mysql_default, layout_complex_key_cache ], + [ source_mysql_default, layout_complex_key_hashed ], + [ source_mysql_default, layout_complex_key_cache ], ]) if not args.no_mongo: @@ -613,16 +687,29 @@ def generate_dictionaries(args): ]) for range_hashed_range_type in range_hashed_range_types: + key_type = "_" + range_hashed_range_type sources_and_layouts.extend([ [ source_file % (generated_prefix + (files[3].format(range_hashed_range_type=range_hashed_range_type))), (layout_range_hashed, range_hashed_range_type) ], - # [ source_clickhouse, layout_range_hashed ], + [ source_clickhouse.format(key_type=key_type), (layout_range_hashed, range_hashed_range_type) ], # [ source_executable, layout_range_hashed ] ]) + if not args.no_mysql: + for range_hashed_range_type in range_hashed_range_types: + key_type = "_" + range_hashed_range_type + source_mysql_typed = source_mysql.format(key_type=key_type) + sources_and_layouts.extend([ + [source_mysql_typed, + (layout_range_hashed, range_hashed_range_type)], + ]) + + dict_name_filter = args.filter.split('/')[0] if args.filter else None for (name, key_idx, has_parent), (source, layout) in zip(dictionaries, sources_and_layouts): + if args.filter and not fnmatch.fnmatch(name, dict_name_filter): + continue + filename = os.path.join(args.generated, 'dictionary_%s.xml' % name) key = keys[key_idx] - if key_idx == 3: layout, range_hashed_range_type = layout # Wrap non-empty type (default) with tag. @@ -670,7 +757,7 @@ def run_tests(args): global SERVER_DIED print "{0:100}".format('Dictionary: ' + dict + ' Name: ' + name + ": "), - if args.filter and not fnmatch.fnmatch(dict, args.filter) and not fnmatch.fnmatch(name, args.filter): + if args.filter and not fnmatch.fnmatch(dict + "/" + name, args.filter): print " ... skipped due to filter." return diff --git a/dbms/tests/integration/test_config_substitutions/configs/config_include_from_env.xml b/dbms/tests/integration/test_config_substitutions/configs/config_include_from_env.xml new file mode 100644 index 00000000000..71e11235749 --- /dev/null +++ b/dbms/tests/integration/test_config_substitutions/configs/config_include_from_env.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + default + default + + + diff --git a/dbms/tests/integration/test_config_substitutions/test.py b/dbms/tests/integration/test_config_substitutions/test.py index 5adeca2b1e9..8472f85a285 100644 --- a/dbms/tests/integration/test_config_substitutions/test.py +++ b/dbms/tests/integration/test_config_substitutions/test.py @@ -9,6 +9,7 @@ node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], e node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], main_configs=['configs/max_query_size.xml']) # include value 77777 node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) +node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, main_configs=['configs/max_query_size.xml']) @pytest.fixture(scope="module") def start_cluster(): @@ -27,6 +28,7 @@ def test_config(start_cluster): assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n" assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n" assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n" + assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" def test_allow_databases(start_cluster): node5.query("CREATE DATABASE db1") diff --git a/dbms/tests/integration/test_union_header/__init__.py b/dbms/tests/integration/test_union_header/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_union_header/configs/remote_servers.xml b/dbms/tests/integration/test_union_header/configs/remote_servers.xml new file mode 100644 index 00000000000..b0bea13abd0 --- /dev/null +++ b/dbms/tests/integration/test_union_header/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/dbms/tests/integration/test_union_header/test.py b/dbms/tests/integration/test_union_header/test.py new file mode 100644 index 00000000000..7b671f03386 --- /dev/null +++ b/dbms/tests/integration/test_union_header/test.py @@ -0,0 +1,44 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in (node1, node2): + + node.query(''' + CREATE TABLE default.t1_local + ( + event_date Date DEFAULT toDate(event_time), + event_time DateTime, + log_type UInt32, + account_id String + ) + ENGINE = MergeTree(event_date, (event_time, account_id), 8192); + ''') + + node.query(''' + CREATE TABLE default.t1 AS default.t1_local + ENGINE = Distributed('two_shards', 'default', 't1_local', rand()); + ''') + + yield cluster + + finally: + cluster.shutdown() + + +def test_read(started_cluster): + assert node1.query('''SELECT event_date, event_time, log_type + FROM default.t1 + WHERE (log_type = 30305) AND (account_id = '111111') + LIMIT 1''').strip() == '' diff --git a/dbms/tests/performance/merge_tree_with_aio/read_hits_with_aio.xml b/dbms/tests/performance/merge_tree_with_aio/read_hits_with_aio.xml new file mode 100644 index 00000000000..e1fb5505577 --- /dev/null +++ b/dbms/tests/performance/merge_tree_with_aio/read_hits_with_aio.xml @@ -0,0 +1,28 @@ + + read_from_hits_with_aio + + + 5000 + 30000 + + + once + + + + + + + + + + hits_1000m_single + + +SELECT count() FROM hits_1000m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; +SELECT count() FROM hits_1000m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; + +SELECT count() FROM hits_1000m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; +SELECT count() FROM hits_1000m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; + + diff --git a/dbms/tests/queries/0_stateless/00417_system_build_options.sh b/dbms/tests/queries/0_stateless/00417_system_build_options.sh index 4de22e36194..4c4b5276a1b 100755 --- a/dbms/tests/queries/0_stateless/00417_system_build_options.sh +++ b/dbms/tests/queries/0_stateless/00417_system_build_options.sh @@ -3,4 +3,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --query="SELECT * FROM system.build_options" | perl -lnE 'print $1 if /(BUILD_DATE|BUILD_TYPE|CXX_COMPILER|CXX_FLAGS|LINK_FLAGS)\s+\S+/'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM system.build_options" | perl -lnE 'print $1 if /(BUILD_DATE|BUILD_TYPE|CXX_COMPILER)\s+\S+/ || /(CXX_FLAGS|LINK_FLAGS)/'; diff --git a/dbms/tests/queries/0_stateless/00428_partition.sh b/dbms/tests/queries/0_stateless/00428_partition.sh index 31e92603024..ce6ad9e1cd8 100755 --- a/dbms/tests/queries/0_stateless/00428_partition.sh +++ b/dbms/tests/queries/0_stateless/00428_partition.sh @@ -18,14 +18,18 @@ $chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(31), 1)" $chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(1), 2)" for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt | wc -l # 2 header lines + 3 columns + # 2 header lines + 3 columns + (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ + cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l done $chl "ALTER TABLE test.partition_428 DETACH PARTITION 197001" $chl "ALTER TABLE test.partition_428 ATTACH PARTITION 197001" for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt | wc -l # 2 header lines + 3 columns + # 2 header lines + 3 columns + (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ + cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l done $chl "ALTER TABLE test.partition_428 MODIFY COLUMN v1 Int8" diff --git a/dbms/tests/queries/0_stateless/00700_decimal_casts.sql b/dbms/tests/queries/0_stateless/00700_decimal_casts.sql index 8ebc98c129e..14593b50b5e 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_casts.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_casts.sql @@ -239,4 +239,8 @@ SELECT toUInt64('2147483649') AS x, toDecimal32(x, 0); -- { serverError 407 } SELECT toUInt64('9223372036854775807') AS x, toDecimal64(x, 0); SELECT toUInt64('9223372036854775809') AS x, toDecimal64(x, 0); -- { serverError 407 } +SELECT toDecimal32(0, rowNumberInBlock()); -- { serverError 44 } +SELECT toDecimal64(0, rowNumberInBlock()); -- { serverError 44 } +SELECT toDecimal128(0, rowNumberInBlock()); -- { serverError 44 } + DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00722_inner_join.reference b/dbms/tests/queries/0_stateless/00722_inner_join.reference index 2fa6ca7df49..9fdac0e26a1 100644 --- a/dbms/tests/queries/0_stateless/00722_inner_join.reference +++ b/dbms/tests/queries/0_stateless/00722_inner_join.reference @@ -1,6 +1,18 @@ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ ┌─x──────┬─name─┐ │ system │ one │ └────────┴──────┘ +┌─database─┬─name─┐ +│ system │ one │ +└──────────┴──────┘ ┌─x──────┬─name─┐ │ system │ one │ └────────┴──────┘ diff --git a/dbms/tests/queries/0_stateless/00722_inner_join.sql b/dbms/tests/queries/0_stateless/00722_inner_join.sql index ab1df6f48b1..9d9c4c48d4e 100644 --- a/dbms/tests/queries/0_stateless/00722_inner_join.sql +++ b/dbms/tests/queries/0_stateless/00722_inner_join.sql @@ -2,12 +2,37 @@ CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.one; CREATE TABLE test.one(dummy UInt8) ENGINE = Memory; +SELECT database, t.name + FROM system.tables AS t + ALL INNER JOIN (SELECT name AS database FROM system.databases) AS db USING database + WHERE database = 'system' AND t.name = 'one' + FORMAT PrettyCompactNoEscapes; + +SELECT database, t.name + FROM (SELECT name AS database FROM system.databases) AS db + ALL INNER JOIN system.tables AS t USING database + WHERE database = 'system' AND t.name = 'one' + FORMAT PrettyCompactNoEscapes; + +SELECT database, t.name + FROM (SELECT name, database FROM system.tables) AS t + ALL INNER JOIN (SELECT name AS database FROM system.databases) AS db USING database + WHERE database = 'system' AND t.name = 'one' + FORMAT PrettyCompactNoEscapes; + SELECT x, t.name FROM (SELECT name, database AS x FROM system.tables) AS t ALL INNER JOIN (SELECT name AS x FROM system.databases) AS db USING x WHERE x = 'system' AND t.name = 'one' FORMAT PrettyCompactNoEscapes; +SELECT database, t.name + FROM (SELECT name, database FROM system.tables) AS t + JOIN (SELECT name AS database FROM system.databases) AS db USING database + WHERE database = 'system' AND t.name = 'one' + SETTINGS join_default_strictness = 'ALL' + FORMAT PrettyCompactNoEscapes; + SELECT x, t.name FROM (SELECT name, database AS x FROM system.tables) AS t JOIN (SELECT name AS x FROM system.databases) AS db USING x diff --git a/dbms/tests/queries/0_stateless/00726_modulo_for_date.sql b/dbms/tests/queries/0_stateless/00726_modulo_for_date.sql index 18f48ad447d..3501f686237 100644 --- a/dbms/tests/queries/0_stateless/00726_modulo_for_date.sql +++ b/dbms/tests/queries/0_stateless/00726_modulo_for_date.sql @@ -1,13 +1,13 @@ -SELECT toDate('2018-06-21') % 234 = toInt16(toDate('2018-06-21')) % 234; -SELECT toDate('2018-06-21') % 23456 = toInt16(toDate('2018-06-21')) % 23456; -SELECT toDate('2018-06-21') % 12376 = toInt16(toDate('2018-06-21')) % 12376; -SELECT toDateTime('2018-06-21 12:12:12') % 234 = toInt32(toDateTime('2018-06-21 12:12:12')) % 234; -SELECT toDateTime('2018-06-21 12:12:12') % 23456 = toInt32(toDateTime('2018-06-21 12:12:12')) % 23456; -SELECT toDateTime('2018-06-21 12:12:12') % 12376 = toInt32(toDateTime('2018-06-21 12:12:12')) % 12376; +SELECT toDate('2018-06-21') % 234 = toUInt16(toDate('2018-06-21')) % 234; +SELECT toDate('2018-06-21') % 23456 = toUInt16(toDate('2018-06-21')) % 23456; +SELECT toDate('2018-06-21') % 12376 = toUInt16(toDate('2018-06-21')) % 12376; +SELECT toDateTime('2018-06-21 12:12:12') % 234 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 234; +SELECT toDateTime('2018-06-21 12:12:12') % 23456 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 23456; +SELECT toDateTime('2018-06-21 12:12:12') % 12376 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 12376; -SELECT toDate('2018-06-21') % 234.8 = toInt16(toDate('2018-06-21')) % 234.8; -SELECT toDate('2018-06-21') % 23456.8 = toInt16(toDate('2018-06-21')) % 23456.8; -SELECT toDate('2018-06-21') % 12376.8 = toInt16(toDate('2018-06-21')) % 12376.8; -SELECT toDateTime('2018-06-21 12:12:12') % 234.8 = toInt32(toDateTime('2018-06-21 12:12:12')) % 234.8; -SELECT toDateTime('2018-06-21 12:12:12') % 23456.8 = toInt32(toDateTime('2018-06-21 12:12:12')) % 23456.8; -SELECT toDateTime('2018-06-21 12:12:12') % 12376.8 = toInt32(toDateTime('2018-06-21 12:12:12')) % 12376.8; +SELECT toDate('2018-06-21') % 234.8 = toUInt16(toDate('2018-06-21')) % 234.8; +SELECT toDate('2018-06-21') % 23456.8 = toUInt16(toDate('2018-06-21')) % 23456.8; +SELECT toDate('2018-06-21') % 12376.8 = toUInt16(toDate('2018-06-21')) % 12376.8; +SELECT toDateTime('2018-06-21 12:12:12') % 234.8 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 234.8; +SELECT toDateTime('2018-06-21 12:12:12') % 23456.8 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 23456.8; +SELECT toDateTime('2018-06-21 12:12:12') % 12376.8 = toUInt32(toDateTime('2018-06-21 12:12:12')) % 12376.8; diff --git a/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.reference b/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.reference new file mode 100644 index 00000000000..3750f104e7f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.reference @@ -0,0 +1,4 @@ +2 +0 +0 +3 diff --git a/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.sql b/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.sql new file mode 100644 index 00000000000..3e83a2284c9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00746_compile_non_deterministic_function.sql @@ -0,0 +1,20 @@ +SET compile_expressions = 1; +SET min_count_to_compile = 1; + +DROP TABLE IF EXISTS test.time_table; + +CREATE TABLE test.time_table(timecol DateTime, value Int32) ENGINE = MergeTree order by tuple(); + +INSERT INTO test.time_table VALUES (now() - 5, 5), (now() - 3, 3); + +SELECT COUNT() from test.time_table WHERE value < now() - 1 AND value != 0 AND modulo(value, 2) != 0 AND timecol < now() - 1; + +SELECT sleep(3); + +INSERT INTO test.time_table VALUES (now(), 101); + +SELECT sleep(3); + +SELECT COUNT() from test.time_table WHERE value < now() - 1 AND value != 0 AND modulo(value, 2) != 0 AND timecol < now() - 1; + +DROP TABLE IF EXISTS test.time_table; diff --git a/dbms/tests/queries/0_stateless/00746_sql_fuzzy.pl b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.pl new file mode 100755 index 00000000000..f16c5061d56 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.pl @@ -0,0 +1,151 @@ +#!/usr/bin/env perl + +# +# Generate some random sql queries +# + +package sqlfuzz; +use 5.16.0; +use strict; +use warnings; +no if $] >= 5.017011, warnings => 'experimental::smartmatch'; +use Data::Dumper; + +sub shuffle(@) { #@$deck = map{ splice @$deck, rand(@$deck), 1 } 0..$#$deck; + my $deck = shift; + $deck = [$deck, @_] unless ref $deck eq 'ARRAY'; + my $i = @$deck; + while ($i--) { + my $j = int rand($i + 1); + @$deck[$i, $j] = @$deck[$j, $i]; + } + return wantarray ? @$deck : $deck; +} + +sub rand_pick ($) { + my ($arr) = @_; + return $arr->[rand(@$arr)]; +} + +sub rand_word { + my ($l) = shift || 8; + my $minl = 1; + @_ = ('a' .. 'z') unless @_; + join '', @_[map { rand @_ } $minl .. $minl + rand($l)]; +} + +sub rand_string { + my ($w) = shift || 5; + join ' ', map { rand_word((), @_) } 1 .. rand($w); +} + +sub one ($$) { + my ($state, $value) = @_; + return ref $value ~~ 'CODE' ? $value->() : $value; +} + +sub one_of ($$) { + my ($state, $hash) = @_; + #state $last_selected; + #my $last_n = $last_selected->{"$hash"}; + my $value; + if ('ARRAY' ~~ ref $hash) { + $value = rand_pick $hash; + } else { + my $keys_array = [sort keys %$hash]; + $value = $hash->{rand_pick $keys_array}; + } + +#$last_selected->{"$hash"} = !exists $last_selected->{"$hash"} ? 0 : $last_selected->{"$hash"} < @keys_array - 1 ? $last_selected->{"$hash"} + 1 : 0; + ##warn join ' : ',$last_selected->{"$hash"}, $keys_array[$last_selected->{"$hash"}], $hash->{$keys_array[$last_selected->{"$hash"}]}; + #my $value = $hash->{$keys_array[$last_selected->{"$hash"}]}; + + return one($state, $value); +} + +sub list_of ($$@) { + my ($state, $opt) = (shift, shift); + my $max = ($opt->{min} // 1) + int rand($opt->{max} || 4); + my @ret; + while (@ret < $max) { + for my $hash (shuffle \@_) { + push @ret, one_of($state, $hash); + } + } + return join ', ', @ret; +} + +sub file_read ($) { + open my $f, '<', $_[0] or return; + local $/ = undef; + my $ret = <$f>; + close $f; + return $ret; +} + +our ($query, $query_select, $expression_cast, $expression, $type, $type_cast, $functions, $table_functions); +$type_cast = {map { $_ => $_ } qw(DateTime Date String)}; +$type = {%$type_cast, (map { $_ => $_ } qw(Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64))}; +$type->{"Nullable($_)"} = "Nullable($_)" for values %$type; +# AS, LIKE, NOT LIKE, IN, NOT IN, GLOBAL IN, GLOBAL NOT IN, BETWEEN, IS, ClosingRoundBracket, Comma, Dot, Arrow, QuestionMark, OR, AND + +$expression_cast = { + 'CAST' => sub { my ($state) = @_; '(CAST((' . one_of($state, $expression_cast) . ') AS ' . one_of($state, $type_cast) . '))' }, + 'SELECT' => sub { + my ($state) = @_; + list_of( + $state, {max => 2}, + #[sub { '( ' . $query->{SELECT}->() . ' ) AS ' . rand_word() }, sub { '( ' . $query->{SELECT}->() . ' ) ' }] + [sub { '( ' . one_of($state, $query_select) . ' ) AS ' . rand_word() }, sub { '( ' . one_of($state, $query_select) . ' ) ' }] + ); + }, + 'number' => sub { my ($state) = @_; return rand_pick(['', '-']) . rand_word(8, 0 .. 9) . rand_pick(['', '.' . rand_word(6, 0 .. 9)]) }, + 'string' => sub { + my ($state) = @_; + return q{'} . rand_word(8, map { $_ ~~ q{'} ? '\\' . $_ : $_ } map {chr} 32 .. 127) . q{'}; + }, + '[]' => '[]', + '[x]' => sub { my ($state) = @_; return '[' . one_of($state, $expression) . ']' }, + 'function()' => + sub { my ($state) = @_; return one_of($state, $functions) . '(' . list_of($state, {min => 0, max => 3}, $expression) . ')' }, + "'\\0'" => "'\\0'", + "''" => "''", + 'NULL' => 'NULL', +}; +$expression = { + %$expression_cast, +}; +$query_select = { + 'SELECT' => sub { my ($state) = @_; return 'SELECT ' . list_of($state, {max => 5}, $expression) }, + 'SELECT function()' => sub { my ($state) = @_; return 'SELECT ' . one($state, $expression->{'function()'}) }, + 'SELECT table_function()' => sub { + my ($state) = @_; + return 'SELECT * FROM ' . one_of($state, $table_functions) . '(' . list_of($state, {min => 0, max => 3}, $expression) . ')'; + }, +}; + +$query = {%$query_select}; + +sub main { + srand($ENV{SQL_FUZZY_SRAND} + $ENV{SQL_FUZZY_RUN}) if $ENV{SQL_FUZZY_SRAND}; + # select name from system.functions format TSV; + $functions = + [split /[\s;,]+/, + file_read($ENV{SQL_FUZZY_FILE_FUNCTIONS} || 'clickhouse-functions') + || '__inner_restore_projection__ __inner_build_projection_composition__ convertCharset one_or_zero findClusterValue findClusterIndex toNullable coalesce isNotNull pointInEllipses transform pow acos asin tan cos tgamma lgamma erfc erf sqrt log10 exp10 e visitParamExtractFloat visitParamExtractUInt decodeURLComponent cutURLParameter cutQueryStringAndFragment cutFragment cutWWW URLPathHierarchy URLHierarchy extractURLParameterNames extractURLParameter queryStringAndFragment pathFull sin topLevelDomain domainWithoutWWW domain protocol greatCircleDistance extract match positionCaseInsensitiveUTF8 positionCaseInsensitive positionUTF8 position replaceRegexpAll replaceRegexpOne arrayStringConcat splitByString splitByChar alphaTokens endsWith startsWith appendTrailingCharIfAbsent substringUTF8 concatAssumeInjective reverseUTF8 upperUTF8 __inner_project__ upper lower length notEmpty trunc round roundAge roundDuration roundToExp2 reinterpretAsString reinterpretAsDateTime reinterpretAsDate reinterpretAsFloat64 reinterpretAsFloat32 reinterpretAsInt64 reinterpretAsInt8 reinterpretAsUInt32 toStartOfFiveMinute toISOYear toISOWeek concat toDecimal64 ifNull toStartOfDay toSecond addSeconds sleepEachRow materialize visitParamExtractInt toStartOfMinute toDayOfWeek toDayOfMonth bitShiftLeft emptyArrayUInt8 parseDateTimeBestEffort toTime toDateTimeOrNull toFloat32OrNull toInt16 IPv6NumToString atan substring arrayIntersect isInfinite toRelativeHourNum hex arrayEnumerateDense toUInt8OrZero toRelativeSecondNum toUInt64OrNull MACNumToString toInt32OrNull toDayOfYear toUnixTimestamp toString toDateOrZero subtractDays toMinute murmurHash3_64 murmurHash2_32 toUInt64 toUInt8 dictGetDateTime empty isFinite caseWithoutExpression caseWithoutExpr visitParamExtractRaw queryString dictGetInt32OrDefault caseWithExpression toInt8OrZero multiIf if intExp10 bitShiftRight less toUInt8OrNull toInt8OrNull bitmaskToArray toIntervalYear toFloat64OrZero dateDiff generateUUIDv4 arrayPopBack toIntervalMonth toUUID notEquals toInt16OrNull murmurHash2_64 hasAny toIntervalMinute isNull tupleElement replaceAll parseDateTimeBestEffortOrZero toFloat32OrZero lowerUTF8 notIn gcd like regionToPopulation MACStringToOUI notLike toStringCutToZero lcm parseDateTimeBestEffortOrNull not toInt32OrZero arrayFilter toInt16OrZero range equals now toTypeName toUInt32OrNull emptyArrayString dictGetDateTimeOrDefault bitRotateRight cutIPv6 toUInt32OrZero timezone reverse runningDifferenceStartingWithFirstValue toDateTime arrayPopFront toInt32 intHash64 extractURLParameters lowCardinalityIndices toStartOfMonth toYear hasAll rowNumberInAllBlocks bitTestAll arrayCount arraySort abs bitNot intDiv intDivOrZero firstSignificantSubdomain dictGetFloat32OrDefault reinterpretAsUInt16 toHour minus regionToArea unhex IPv4StringToNum toIntervalHour toInt8 dictGetFloat32 log IPv4NumToString modulo arrayEnumerate cutQueryString reinterpretAsFixedString countEqual bitTest toDecimal128 plus or reinterpretAsUInt64 toMonth visitParamExtractBool emptyArrayUInt64 replaceOne arrayReverseSort toFloat32 toRelativeMonthNum emptyArrayInt32 toRelativeYearNum arrayElement log2 array arrayReverse toUInt64OrZero emptyArrayFloat64 negate arrayPushBack subtractWeeks bitTestAny bitAnd toDecimal32 arrayPushFront lessOrEquals intExp2 toUInt16OrZero arrayConcat arrayCumSum arraySlice addDays dictGetUInt8 toUInt32 bitOr caseWithExpr toStartOfYear toIntervalDay MD5 emptyArrayUInt32 emptyArrayInt8 toMonday addMonths arrayUniq SHA256 arrayExists multiply toUInt16OrNull dictGetInt8 visitParamHas emptyArrayInt64 toIntervalSecond toDate sleep emptyArrayToSingle path toInt64OrZero SHA1 extractAll emptyArrayDate dumpColumnStructure toInt64 lengthUTF8 greatest arrayEnumerateUniq arrayDistinct arrayFirst toFixedString IPv4NumToStringClassC toFloat64OrNull IPv4ToIPv6 identity ceil toStartOfQuarter dictGetInt8OrDefault MACStringToNum emptyArrayUInt16 UUIDStringToNum dictGetUInt16 toStartOfFifteenMinutes toStartOfHour sumburConsistentHash toStartOfISOYear toRelativeQuarterNum toRelativeWeekNum toRelativeDayNum cbrt yesterday bitXor timeSlot timeSlots emptyArrayInt16 dictGetInt16 toYYYYMM toYYYYMMDDhhmmss toUInt16 addMinutes addHours addWeeks nullIf subtractSeconds subtractMinutes toIntervalWeek subtractHours isNaN subtractMonths toDateOrNull subtractYears toTimeZone formatDateTime has cityHash64 intHash32 fragment regionToCity indexOf regionToDistrict regionToCountry visibleWidth regionToContinent regionToTopContinent toColumnTypeName regionHierarchy CHAR_LENGTH least divide SEHierarchy dictGetDate OSToRoot SEToRoot OSIn SEIn regionToName dictGetStringOrDefault OSHierarchy exp floor dictGetUInt8OrDefault dictHas dictGetUInt64 cutToFirstSignificantSubdomain dictGetInt32 pointInPolygon dictGetInt64 blockNumber IPv6StringToNum dictGetString dictGetFloat64 dictGetUUID CHARACTER_LENGTH toQuarter dictGetHierarchy toFloat64 arraySum toInt64OrNull dictIsIn dictGetUInt16OrDefault dictGetUInt32OrDefault emptyArrayDateTime greater jumpConsistentHash dictGetUInt64OrDefault dictGetInt16OrDefault dictGetInt64OrDefault reinterpretAsInt32 dictGetUInt32 murmurHash3_32 bar dictGetUUIDOrDefault rand modelEvaluate arrayReduce farmHash64 bitmaskToList formatReadableSize halfMD5 SHA224 arrayMap sipHash64 dictGetFloat64OrDefault sipHash128 metroHash64 murmurHash3_128 yandexConsistentHash emptyArrayFloat32 arrayAll toYYYYMMDD today arrayFirstIndex greaterOrEquals arrayDifference visitParamExtractString toDateTimeOrZero globalNotIn throwIf and xor currentDatabase hostName URLHash getSizeOfEnumType defaultValueOfArgumentType blockSize tuple arrayCumSumNonNegative rowNumberInBlock arrayResize ignore toRelativeMinuteNum indexHint reinterpretAsInt16 addYears arrayJoin replicate hasColumnInTable version regionIn uptime runningAccumulate runningDifference assumeNotNull pi finalizeAggregation toLowCardinality exp2 lowCardinalityKeys in globalIn dictGetDateOrDefault rand64 CAST bitRotateLeft randConstant UUIDNumToString reinterpretAsUInt8 truncate ceiling retention maxIntersections groupBitXor groupBitOr uniqUpTo uniqCombined uniqExact uniq covarPop stddevPop varPop covarSamp varSamp sumMap corrStable corr quantileTiming quantileDeterministic quantilesExact uniqHLL12 quantilesTiming covarPopStable stddevSampStable quantilesExactWeighted quantileExactWeighted quantileTimingWeighted quantileExact quantilesDeterministic quantiles topK sumWithOverflow count groupArray stddevSamp groupArrayInsertAt quantile quantilesTimingWeighted quantileTDigest quantilesTDigest windowFunnel min argMax varSampStable maxIntersectionsPosition quantilesTDigestWeighted groupUniqArray sequenceCount sumKahan any anyHeavy histogram quantileTDigestWeighted max groupBitAnd argMin varPopStable avg sequenceMatch stddevPopStable sum anyLast covarSampStable BIT_XOR medianExactWeighted medianTiming medianExact median medianDeterministic VAR_SAMP STDDEV_POP medianTDigest VAR_POP medianTDigestWeighted BIT_OR STDDEV_SAMP medianTimingWeighted COVAR_SAMP COVAR_POP BIT_AND' + ]; + $functions = [grep { not $_ ~~ [qw(__inner_restore_projection__ extractURLParameter globalNotIn globalIn)] } @$functions]; # will be removed + # select name from system.table_functions format TSV; + $table_functions = + [split /[\s;,]+/, + file_read($ENV{SQL_FUZZY_FILE_TABLE_FUNCTIONS} || 'clickhouse-table-functions') + || 'mysql jdbc odbc remote catBoostPool merge file cluster shardByHash url numbers']; + $table_functions = [grep { not $_ ~~ [qw(numbers)] } @$table_functions]; # too slow + say one_of({}, $query), ';' for 1 .. ($ENV{SQL_FUZZY_LINES} || 100); +} + +main() unless caller; + +#say rand_word() for 1..10000; +#say rand_word(8, 0..9) for 1..1000; + diff --git a/dbms/tests/queries/0_stateless/00746_sql_fuzzy.reference b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.reference new file mode 100644 index 00000000000..7193c3d3f3d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.reference @@ -0,0 +1 @@ +Still alive diff --git a/dbms/tests/queries/0_stateless/00746_sql_fuzzy.sh b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.sh new file mode 100755 index 00000000000..913963c7318 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +export SQL_FUZZY_FILE_FUNCTIONS=${CLICKHOUSE_TMP}/clickhouse-functions +$CLICKHOUSE_CLIENT -q "select name from system.functions format TSV;" > $SQL_FUZZY_FILE_FUNCTIONS + +export SQL_FUZZY_FILE_TABLE_FUNCTIONS=${CLICKHOUSE_TMP}/clickhouse-table_functions +$CLICKHOUSE_CLIENT -q "select name from system.table_functions format TSV;" > $SQL_FUZZY_FILE_TABLE_FUNCTIONS + +# This is short run for ordinary tests. +# if you want long run use: env SQL_FUZZY_RUNS=100000 clickhouse-test sql_fuzzy + +for SQL_FUZZY_RUN in $(seq ${SQL_FUZZY_RUNS:=10}); do + env SQL_FUZZY_RUN=$SQL_FUZZY_RUN $CURDIR/00746_sql_fuzzy.pl | $CLICKHOUSE_CLIENT -n --ignore-error >/dev/null 2>&1 + if [[ `$CLICKHOUSE_CLIENT -q "SELECT 'Still alive'"` != 'Still alive' ]]; then + break + fi +done + +$CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" + +# Query replay: +# cat clickhouse-server.log | grep -aF " executeQuery: (from " | perl -lpe 's/^.*executeQuery: \(from \S+\) (.*)/$1;/' | clickhouse-client -n --ignore-error diff --git a/dbms/tests/queries/0_stateless/00747_contributors.reference b/dbms/tests/queries/0_stateless/00747_contributors.reference new file mode 100644 index 00000000000..9766475a418 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00747_contributors.reference @@ -0,0 +1 @@ +ok diff --git a/dbms/tests/queries/0_stateless/00747_contributors.sql b/dbms/tests/queries/0_stateless/00747_contributors.sql new file mode 100644 index 00000000000..ec75d877841 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00747_contributors.sql @@ -0,0 +1,2 @@ +-- Normally table should contain 250+ contributors. But when fast git clone used (--depth=X) (Travis build) table will contain only <=X contributors +SELECT if ((SELECT count(*) FROM system.contributors) > 1, 'ok', 'fail'); diff --git a/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.reference b/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.reference new file mode 100644 index 00000000000..7b39e1997c1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.reference @@ -0,0 +1 @@ +1 hello diff --git a/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.sql b/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.sql new file mode 100644 index 00000000000..f2cf8579bab --- /dev/null +++ b/dbms/tests/queries/0_stateless/00749_inner_join_of_unnamed_subqueries.sql @@ -0,0 +1,40 @@ +DROP TABLE IF EXISTS test.left_table; +DROP TABLE IF EXISTS test.right_table; + +CREATE TABLE test.left_table(APIKey Int32, SomeColumn String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO test.left_table VALUES(1, 'somestr'); + +CREATE TABLE test.right_table(APIKey Int32, EventValueForPostback String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO test.right_table VALUES(1, 'hello'), (2, 'WORLD'); + +SELECT + APIKey, + ConversionEventValue +FROM + test.left_table AS left_table +ALL INNER JOIN + ( + SELECT * + FROM + ( + SELECT + APIKey, + EventValueForPostback AS ConversionEventValue + FROM + test.right_table AS right_table + ) + ALL INNER JOIN + ( + SELECT + APIKey + FROM + test.left_table as left_table + GROUP BY + APIKey + ) USING (APIKey) + ) USING (APIKey); + +DROP TABLE IF EXISTS test.left_table; +DROP TABLE IF EXISTS test.right_table; diff --git a/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.reference b/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.reference new file mode 100644 index 00000000000..77a14d12483 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.reference @@ -0,0 +1,3 @@ +2018-10-31 05:05:00 0 +2018-10-31 06:06:00 10 +2018-10-28 10:00:00 20 diff --git a/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.sql b/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.sql new file mode 100644 index 00000000000..5957ec3f50f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00750_merge_tree_merge_with_o_direct.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS test.sample_merge_tree; + +CREATE TABLE test.sample_merge_tree (dt DateTime, x UInt64) ENGINE = MergeTree PARTITION BY toYYYYMMDD(dt) ORDER BY x SETTINGS min_merge_bytes_to_use_direct_io=1, index_granularity = 8192; + +INSERT INTO test.sample_merge_tree VALUES (toDateTime('2018-10-31 05:05:00'), 0), (toDateTime('2018-10-31 06:06:00'), 10), (toDateTime('2018-10-28 10:00:00'), 20); + +OPTIMIZE TABLE test.sample_merge_tree FINAL; + +SELECT * FROM test.sample_merge_tree ORDER BY x; + +DROP TABLE IF EXISTS test.sample_merge_tree; diff --git a/dbms/tests/queries/bugs/fuzzy.sql b/dbms/tests/queries/bugs/fuzzy.sql new file mode 100644 index 00000000000..52008e27ca7 --- /dev/null +++ b/dbms/tests/queries/bugs/fuzzy.sql @@ -0,0 +1,9 @@ +SELECT __inner_restore_projection__(2.0885, -66.72488); +SELECT __inner_restore_projection__(-4, ''); +SELECT __inner_restore_projection__(067274, 'vb\s'); +SELECT sequenceCount((CAST((( SELECT NULL ) AS rg, ( SELECT ( SELECT [], 'A') AS String))]]); diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index e1780db7c6f..3c78bb71978 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -26,4 +26,4 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -CMD ["/usr/bin/clickhouse-client"] +ENTRYPOINT ["/usr/bin/clickhouse-client"] diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index c7268393408..6044d4e3b43 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -53,6 +53,135 @@ FROM ontime Selects the last value encountered. The result is just as indeterminate as for the `any` function. +##groupBitAnd + +Applies bitwise `AND` for series of numbers. + +``` +groupBitAnd(expr) +``` + +**Parameters** + +`expr` – An expression that results in `UInt*` type. + +**Return value** + +Value of the `UInt*` type. + +**Example** + +Test data: + +``` +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +The query: + +``` +SELECT groupBitAnd(num) FROM t +``` + +Where `num` is the column with the test data. + +Result: + +``` +binary decimal +00000100 = 4 +``` + +##groupBitOr + +Applies bitwise `OR` for series of numbers. + +``` +groupBitOr(expr) +``` + +**Parameters** + +`expr` – An expression that results in `UInt*` type. + +**Return value** + +Value of the `UInt*` type. + +**Example** + +Test data: + +``` +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +Query: + +``` +SELECT groupBitOr(num) FROM t +``` + +Where `num` is the column with the test data. + +Result: + +``` +binary decimal +01111101 = 125 +``` + +##groupBitXor + +Applies bitwise `XOR` for series of numbers. + +``` +groupBitXor(expr) +``` + +**Parameters** + +`expr` – An expression that results in `UInt*` type. + +**Return value** + +Value of the `UInt*` type. + +**Example** + +Test data: + +``` +binary decimal +00101100 = 44 +00011100 = 28 +00001101 = 13 +01010101 = 85 +``` + +Query: + +``` +SELECT groupBitXor(num) FROM t +``` + +Where `num` is the column with the test data. + +Result: + +``` +binary decimal +01101000 = 104 +``` + ## min(x) Calculates the minimum. diff --git a/docs/ru/index.md b/docs/ru/index.md index a2f6945bd04..24e97aaff83 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -58,7 +58,7 @@ ClickHouse - столбцовая система управления базам ## Причины, по которым столбцовые СУБД лучше подходят для OLAP сценария -Столбцовые СУБД лучше (от 100 раз по скорости обработки большинства запросов) подходят для OLAP сценария работы. Причины в деталях буду разъяснены ниже, а сам факт проще проще продемонстрировать визуально: +Столбцовые СУБД лучше (от 100 раз по скорости обработки большинства запросов) подходят для OLAP сценария работы. Причины в деталях будут разъяснены ниже, а сам факт проще продемонстрировать визуально: **Строковые СУБД** @@ -76,7 +76,7 @@ ClickHouse - столбцовая система управления базам 2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. 3. За счёт уменьшения ввода-вывода, больше данных влезает в системный кэш. -Для примера, для запроса "посчитать количество записей для каждой рекламной системы", требуется прочитать один столбец "идентификатор рекламной системы", который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается. +Например, для запроса "посчитать количество записей для каждой рекламной системы", требуется прочитать один столбец "идентификатор рекламной системы", который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается.
Пример ``` diff --git a/docs/ru/query_language/dicts/external_dicts_dict_structure.md b/docs/ru/query_language/dicts/external_dicts_dict_structure.md index e26eb0282bc..76271effe9c 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_structure.md +++ b/docs/ru/query_language/dicts/external_dicts_dict_structure.md @@ -60,7 +60,7 @@ ClickHouse поддерживает следующие виды ключей: ### Составной ключ -Ключем может быть кортеж (`tuple`) из полей произвольных типов. [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) в этом случае должен быть `complex_key_hashed` или `complex_key_cache`. +Ключом может быть кортеж (`tuple`) из полей произвольных типов. [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) в этом случае должен быть `complex_key_hashed` или `complex_key_cache`. !!! tip "Совет" Cоставной ключ может состоять из одного элемента. Это даёт возможность использовать в качестве ключа, например, строку. diff --git a/docs/zh/operations/table_engines/index.md b/docs/zh/operations/table_engines/index.md index 994dff9b516..b1d1c5db49d 120000 --- a/docs/zh/operations/table_engines/index.md +++ b/docs/zh/operations/table_engines/index.md @@ -1 +1,13 @@ -../../../en/operations/table_engines/index.md \ No newline at end of file +# 表引擎 +表引擎（即表的类型）决定了： + +* 数据的存储方式和位置，写到哪里以及从哪里读取数据。 +* 支持哪些查询以及如何支持。 +* 并发数据访问。 +* 索引的使用（如果存在）。 +* 是否可以执行多线程请求。 +* 数据复制参数。 + +在读取时，引擎只需要输出所请求的列，但在某些情况下，引擎可以在响应请求时部分处理数据。 + +对于大多数正式的任务，应该使用MergeTree族中的引擎。 diff --git a/libs/libcommon/include/common/readline_use.h b/libs/libcommon/include/common/readline_use.h index 97622b26839..549676ef9b2 100644 --- a/libs/libcommon/include/common/readline_use.h +++ b/libs/libcommon/include/common/readline_use.h @@ -10,7 +10,6 @@ #include #elif USE_LIBEDIT #include - #include // Y_IGNORE #else #include #include diff --git a/release b/release index 23bfd6f2dd6..2a76b9bbdd4 100755 --- a/release +++ b/release @@ -107,6 +107,8 @@ echo -e "\nCurrent version is $VERSION_STRING" gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" +$CURDIR/dbms/src/Storages/System/StorageSystemContributors.sh + if [ -z "$USE_PBUILDER" ] ; then DEB_CC=${DEB_CC:=`which gcc-7 gcc-8 gcc | head -n1`} DEB_CXX=${DEB_CXX:=`which g++-7 g++-8 g++ | head -n1`} diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 2952444b8f7..27b18bfc1fc 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,3 +1,6 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) target_link_libraries(clickhouse-zookeeper-cli clickhouse_common_zookeeper ${LINE_EDITING_LIBS}) +if (READLINE_INCLUDE_DIR) + target_include_directories (clickhouse-zookeeper-cli SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) +endif () INSTALL(TARGETS clickhouse-zookeeper-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-utils)