From e19c3b3a462ece53b140acdd1271dc02481b07da Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 11 Dec 2017 15:07:26 +0300 Subject: [PATCH 1/3] Headers markup is unified through the document. --- docs/ru/agg_functions/combinators.md | 21 +-- docs/ru/agg_functions/parametric_functions.md | 12 +- docs/ru/data_types/array.md | 3 +- docs/ru/data_types/boolean.md | 3 +- docs/ru/data_types/date.md | 3 +- docs/ru/data_types/datetime.md | 6 +- docs/ru/data_types/enum.md | 3 +- docs/ru/data_types/fixedstring.md | 3 +- docs/ru/data_types/index.md | 3 +- docs/ru/data_types/int_uint.md | 7 +- .../aggregatefunction.md | 2 +- .../nested_data_structures/index.md | 3 +- .../nested_data_structures/nested.md | 2 +- .../special_data_types/expression.md | 2 +- .../ru/data_types/special_data_types/index.md | 3 +- docs/ru/data_types/special_data_types/set.md | 2 +- docs/ru/data_types/string.md | 5 +- docs/ru/data_types/tuple.md | 3 +- docs/ru/development/architecture.md | 54 +++----- docs/ru/development/build.md | 24 ++-- docs/ru/development/build_osx.md | 18 +-- docs/ru/development/index.md | 3 +- docs/ru/development/tests.md | 6 +- docs/ru/dicts/external_dicts.md | 3 +- docs/ru/dicts/external_dicts_dict.md | 3 +- docs/ru/dicts/external_dicts_dict_layout.md | 24 ++-- docs/ru/dicts/external_dicts_dict_lifetime.md | 3 +- docs/ru/dicts/external_dicts_dict_sources.md | 21 +-- .../ru/dicts/external_dicts_dict_structure.md | 9 +- docs/ru/dicts/index.md | 3 +- docs/ru/dicts/internal_dicts.md | 3 +- docs/ru/formats/capnproto.md | 3 +- docs/ru/formats/csv.md | 3 +- docs/ru/formats/csvwithnames.md | 3 +- docs/ru/formats/index.md | 3 +- docs/ru/formats/json.md | 3 +- docs/ru/formats/jsoncompact.md | 3 +- docs/ru/formats/jsoneachrow.md | 3 +- docs/ru/formats/native.md | 3 +- docs/ru/formats/null.md | 3 +- docs/ru/formats/pretty.md | 3 +- docs/ru/formats/prettycompact.md | 3 +- docs/ru/formats/prettycompactmonoblock.md | 3 +- docs/ru/formats/prettynoescapes.md | 9 +- docs/ru/formats/prettyspace.md | 3 +- docs/ru/formats/rowbinary.md | 3 +- docs/ru/formats/tabseparated.md | 3 +- docs/ru/formats/tabseparatedraw.md | 3 +- docs/ru/formats/tabseparatedwithnames.md | 3 +- .../formats/tabseparatedwithnamesandtypes.md | 3 +- docs/ru/formats/tskv.md | 3 +- docs/ru/formats/values.md | 3 +- docs/ru/formats/vertical.md | 3 +- docs/ru/formats/xml.md | 3 +- docs/ru/functions/arithmetic_functions.md | 36 ++--- docs/ru/functions/array_join.md | 3 +- docs/ru/functions/bit_functions.md | 15 +-- docs/ru/functions/comparison_functions.md | 16 +-- docs/ru/functions/conditional_functions.md | 5 +- docs/ru/functions/date_time_functions.md | 57 ++++---- docs/ru/functions/encoding_functions.md | 15 +-- docs/ru/functions/ext_dict_functions.md | 23 ++-- docs/ru/functions/hash_functions.md | 25 ++-- docs/ru/functions/higher_order_functions.md | 6 +- docs/ru/functions/in_functions.md | 9 +- docs/ru/functions/index.md | 27 ++-- docs/ru/functions/ip_address_functions.md | 11 +- docs/ru/functions/json_functions.md | 17 ++- docs/ru/functions/logical_functions.md | 11 +- docs/ru/functions/math_functions.md | 45 +++---- docs/ru/functions/other_functions.md | 51 ++++--- docs/ru/functions/random_functions.md | 7 +- docs/ru/functions/rounding_functions.md | 15 +-- .../functions/splitting_merging_functions.md | 11 +- docs/ru/functions/string_functions.md | 33 +++-- docs/ru/functions/string_replace_functions.md | 11 +- docs/ru/functions/string_search_functions.md | 17 ++- .../ru/functions/type_conversion_functions.md | 31 +++-- docs/ru/functions/url_functions.md | 53 ++++---- docs/ru/functions/ym_dict_functions.md | 6 +- .../example_datasets/amplab_benchmark.md | 3 +- .../example_datasets/criteo.md | 3 +- .../example_datasets/nyc_taxi.md | 15 +-- .../example_datasets/star_schema.md | 3 +- .../example_datasets/wikistat.md | 3 +- docs/ru/getting_started/index.md | 12 +- docs/ru/index.md | 3 +- docs/ru/interfaces/http_interface.md | 6 +- docs/ru/interfaces/index.md | 3 +- docs/ru/interfaces/jdbc.md | 3 +- docs/ru/interfaces/tcp.md | 3 +- .../third-party_client_libraries.md | 3 +- docs/ru/interfaces/third-party_gui.md | 3 +- .../features_considered_disadvantages.md | 3 +- docs/ru/introduction/performance.md | 15 +-- .../introduction/possible_silly_questions.md | 6 +- docs/ru/introduction/what_is_clickhouse.md | 3 +- docs/ru/introduction/ya_metrika_task.md | 9 +- docs/ru/operations/access_rights.md | 3 +- docs/ru/operations/configuration_files.md | 3 +- docs/ru/operations/index.md | 3 +- docs/ru/operations/quotas.md | 3 +- docs/ru/operations/server_settings/index.md | 3 +- .../ru/operations/server_settings/settings.md | 110 ++++++--------- docs/ru/operations/settings/index.md | 3 +- .../operations/settings/query_complexity.md | 102 +++++--------- docs/ru/operations/settings/settings.md | 120 ++++++----------- .../operations/settings/settings_profiles.md | 3 +- docs/ru/operations/tips.md | 45 +++---- docs/ru/operators/index.md | 51 +++---- docs/ru/query_language/clickhouse_local.md | 3 +- docs/ru/query_language/index.md | 3 +- docs/ru/query_language/queries.md | 127 +++++++----------- docs/ru/query_language/syntax.md | 36 ++--- docs/ru/roadmap.md | 12 +- docs/ru/system_tables/index.md | 3 +- .../system.asynchronous_metrics.md | 3 +- docs/ru/system_tables/system.clusters.md | 3 +- docs/ru/system_tables/system.columns.md | 3 +- docs/ru/system_tables/system.databases.md | 3 +- docs/ru/system_tables/system.dictionaries.md | 3 +- docs/ru/system_tables/system.events.md | 3 +- docs/ru/system_tables/system.functions.md | 3 +- docs/ru/system_tables/system.merges.md | 3 +- docs/ru/system_tables/system.metrics.md | 3 +- docs/ru/system_tables/system.numbers.md | 3 +- docs/ru/system_tables/system.numbers_mt.md | 3 +- docs/ru/system_tables/system.one.md | 3 +- docs/ru/system_tables/system.parts.md | 3 +- docs/ru/system_tables/system.processes.md | 3 +- docs/ru/system_tables/system.replicas.md | 3 +- docs/ru/system_tables/system.settings.md | 3 +- docs/ru/system_tables/system.tables.md | 3 +- docs/ru/system_tables/system.zookeeper.md | 3 +- docs/ru/table_engines/aggregatingmergetree.md | 3 +- docs/ru/table_engines/buffer.md | 3 +- docs/ru/table_engines/collapsingmergetree.md | 3 +- .../table_engines/custom_partitioning_key.md | 3 +- docs/ru/table_engines/distributed.md | 3 +- docs/ru/table_engines/external_data.md | 3 +- docs/ru/table_engines/file.md | 3 +- docs/ru/table_engines/graphitemergetree.md | 6 +- docs/ru/table_engines/index.md | 3 +- docs/ru/table_engines/join.md | 3 +- docs/ru/table_engines/log.md | 3 +- docs/ru/table_engines/materializedview.md | 3 +- docs/ru/table_engines/memory.md | 3 +- docs/ru/table_engines/merge.md | 6 +- docs/ru/table_engines/mergetree.md | 3 +- docs/ru/table_engines/null.md | 3 +- docs/ru/table_engines/replacingmergetree.md | 3 +- docs/ru/table_engines/replication.md | 29 ++-- docs/ru/table_engines/set.md | 3 +- docs/ru/table_engines/summingmergetree.md | 3 +- docs/ru/table_engines/tinylog.md | 3 +- docs/ru/table_engines/view.md | 3 +- docs/ru/table_functions/index.md | 3 +- docs/ru/table_functions/merge.md | 3 +- docs/ru/table_functions/remote.md | 3 +- 159 files changed, 679 insertions(+), 1105 deletions(-) diff --git a/docs/ru/agg_functions/combinators.md b/docs/ru/agg_functions/combinators.md index b9272b41f5a..ba08f46e4ea 100644 --- a/docs/ru/agg_functions/combinators.md +++ b/docs/ru/agg_functions/combinators.md @@ -1,12 +1,10 @@ -Комбинаторы агрегатных функций -============================== +# Комбинаторы агрегатных функций К имени агрегатной функции может быть приписан некоторый суффикс. При этом, работа агрегатной функции некоторым образом модифицируется. --If ---- +## -If К имени любой агрегатной функции может быть приписан суффикс -If. В этом случае, агрегатная функция принимает ещё один дополнительный аргумент - условие (типа UInt8). Агрегатная функция будет обрабатывать только те строки, для которых условие сработало. Если условие ни разу не сработало - возвращается некоторое значение по умолчанию (обычно - нули, пустые строки). @@ -15,8 +13,7 @@ С помощью условных агрегатных функций, вы можете вычислить агрегаты сразу для нескольких условий, не используя подзапросы и `JOIN`-ы. Например, в Яндекс.Метрике, условные агрегатные функции используются для реализации функциональности сравнения сегментов. --Array ------- +## -Array К имени любой агрегатной функции может быть приписан суффикс -Array. В этом случае, агрегатная функция вместо аргументов типов T принимает аргументы типов Array(T) (массивы). Если агрегатная функция принимает несколько аргументов, то это должны быть массивы одинаковых длин. При обработке массивов, агрегатная функция работает, как исходная агрегатная функция по всем элементам массивов. @@ -26,22 +23,18 @@ Комбинаторы -If и -Array можно сочетать. При этом, должен сначала идти Array, а потом If. Примеры: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. Из-за такого порядка получается, что аргумент cond не должен быть массивом. --State ------- +## -State В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции uniq - количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` - хэш-таблицу для рассчёта количества уникальных значений), которое имеет тип AggregateFunction(...) и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации - смотрите разделы «AggregatingMergeTree» и «функции для работы с промежуточными состояниями агрегации». --Merge ------- +## -Merge В случае применения этого комбинатора, агрегатная функция будет принимать в качестве аргумента промежуточное состояние агрегации, доагрегировать (объединять вместе) эти состояния, и возвращать готовое значение. --MergeState. ------------- +## -MergeState. Выполняет слияние промежуточных состояний агрегации, аналогично комбинатору -Merge, но возвращает не готовое значение, а промежуточное состояние агрегации, аналогично комбинатору -State. --ForEach --------- +## -ForEach Преобразует агрегатную функцию для таблиц в агрегатную функцию для массивов, которая применяет агрегирование для соответствующих элементов массивов и возвращает массив результатов. Например, `sumForEach` для массивов `[1, 2]`, `[3, 4, 5]` и `[6, 7]` даст результат `[10, 13, 5]`, сложив соответственные элементы массивов. diff --git a/docs/ru/agg_functions/parametric_functions.md b/docs/ru/agg_functions/parametric_functions.md index 90df031ab65..4a1c1ee7d38 100644 --- a/docs/ru/agg_functions/parametric_functions.md +++ b/docs/ru/agg_functions/parametric_functions.md @@ -1,12 +1,10 @@ -Параметрические агрегатные функции -================================== +# Параметрические агрегатные функции Некоторые агрегатные функции могут принимать не только столбцы-аргументы (по которым производится свёртка), но и набор параметров - констант для инициализации. Синтаксис - две пары круглых скобок вместо одной. Первая - для параметров, вторая - для аргументов. -sequenceMatch(pattern)(time, cond1, cond2, ...) ------------------------------------------------ +## sequenceMatch(pattern)(time, cond1, cond2, ...) Сопоставление с образцом для цепочки событий. @@ -47,14 +45,12 @@ minIf(EventTime, URL LIKE '%company%') < maxIf(EventTime, URL LIKE '%cart%'). События, произошедшие в одну секунду, могут оказаться в цепочке в произвольном порядке. От этого может зависеть результат работы функции. -sequenceCount(pattern)(time, cond1, cond2, ...) ------------------------------------------------ +## sequenceCount(pattern)(time, cond1, cond2, ...) Аналогично функции sequenceMatch, но возвращает не факт наличия цепочки событий, а UInt64 - количество найденных цепочек. Цепочки ищутся без перекрытия. То есть, следующая цепочка может начаться только после окончания предыдущей. -uniqUpTo(N)(x) --------------- +## uniqUpTo(N)(x) Вычисляет количество различных значений аргумента, если оно меньше или равно N. В случае, если количество различных значений аргумента больше N, возвращает N + 1. diff --git a/docs/ru/data_types/array.md b/docs/ru/data_types/array.md index 2dce5a616f5..894b6c7647c 100644 --- a/docs/ru/data_types/array.md +++ b/docs/ru/data_types/array.md @@ -1,5 +1,4 @@ -Array(T) --------- +# Array(T) Массив из элементов типа T. Типом T может быть любой тип, в том числе, массив. Многомерные массивы не рекомендуется использовать, так как их поддержка довольно слабая (например, многомерные массивы нельзя сохранить в таблицы с движком семейства MergeTree). diff --git a/docs/ru/data_types/boolean.md b/docs/ru/data_types/boolean.md index 484373e5cd5..541f8ef0345 100644 --- a/docs/ru/data_types/boolean.md +++ b/docs/ru/data_types/boolean.md @@ -1,4 +1,3 @@ -Булевы значения ---------------- +# Булевы значения Отдельного типа для булевых значений нет. Для них используется тип UInt8, в котором используются только значения 0 и 1. diff --git a/docs/ru/data_types/date.md b/docs/ru/data_types/date.md index cb5446de3e7..4f18a786f31 100644 --- a/docs/ru/data_types/date.md +++ b/docs/ru/data_types/date.md @@ -1,5 +1,4 @@ -Date ----- +# Date Дата. Хранится в двух байтах в виде (беззнакового) числа дней, прошедших от 1970-01-01. Позволяет хранить значения от чуть больше, чем начала unix-эпохи до верхнего порога, определяющегося константой на этапе компиляции (сейчас - до 2038 года, но может быть расширено до 2106 года). Минимальное значение выводится как 0000-00-00. diff --git a/docs/ru/data_types/datetime.md b/docs/ru/data_types/datetime.md index 700a2d03f40..8b0ca24d9dd 100644 --- a/docs/ru/data_types/datetime.md +++ b/docs/ru/data_types/datetime.md @@ -1,10 +1,10 @@ -DateTime --------- +# DateTime + Дата-с-временем. Хранится в 4 байтах, в виде (беззнакового) unix timestamp. Позволяет хранить значения в том же интервале, что и для типа Date. Минимальное значение выводится как 0000-00-00 00:00:00. Время хранится с точностью до одной секунды (без учёта секунд координации). -### Часовые пояса +## Часовые пояса Дата-с-временем преобразуется из текстового (разбитого на составляющие) в бинарный вид и обратно, с использованием системного часового пояса на момент старта клиента или сервера. В текстовом виде, теряется информация о том, был ли произведён перевод стрелок. diff --git a/docs/ru/data_types/enum.md b/docs/ru/data_types/enum.md index dc470531616..ff977431959 100644 --- a/docs/ru/data_types/enum.md +++ b/docs/ru/data_types/enum.md @@ -1,5 +1,4 @@ -Enum ----- +# Enum Enum8 или Enum16. Представляет собой конечное множество строковых значений, сохраняемых более эффективно, чем это делает тип данных `String`. diff --git a/docs/ru/data_types/fixedstring.md b/docs/ru/data_types/fixedstring.md index b3e558f5a9b..75fb9650b02 100644 --- a/docs/ru/data_types/fixedstring.md +++ b/docs/ru/data_types/fixedstring.md @@ -1,5 +1,4 @@ -FixedString(N) --------------- +# FixedString(N) Строка фиксированной длины N байт (не символов, не кодовых точек). N должно быть строго положительным натуральным числом. При чтении сервером строки (например, при парсинге данных для INSERT), содержащей меньшее число байт, строка дополняется до N байт дописыванием нулевых байт справа. diff --git a/docs/ru/data_types/index.md b/docs/ru/data_types/index.md index 1e327013035..49c6d31c5fe 100644 --- a/docs/ru/data_types/index.md +++ b/docs/ru/data_types/index.md @@ -1,7 +1,6 @@ -Типы данных -=========== +# Типы данных ```eval_rst .. toctree:: diff --git a/docs/ru/data_types/int_uint.md b/docs/ru/data_types/int_uint.md index ff12c0d4e40..d79ce7326bc 100644 --- a/docs/ru/data_types/int_uint.md +++ b/docs/ru/data_types/int_uint.md @@ -1,9 +1,8 @@ -UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 --------------------------------------------------------- +# UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 Целые числа фиксированной длины, без знака или со знаком. -#### Диапазоны Int +## Диапазоны Int - Int8 - [ -128 : 127 ] - Int16 - [ -32768 : 32767 ] @@ -12,7 +11,7 @@ UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 -#### Диапазоны Uint +## Диапазоны Uint - UInt8 - [ 0 : 255 ] - UInt16 - [ 0 : 65535 ] diff --git a/docs/ru/data_types/nested_data_structures/aggregatefunction.md b/docs/ru/data_types/nested_data_structures/aggregatefunction.md index 8d29548a9b3..a15205cf120 100644 --- a/docs/ru/data_types/nested_data_structures/aggregatefunction.md +++ b/docs/ru/data_types/nested_data_structures/aggregatefunction.md @@ -1,3 +1,3 @@ -### AggregateFunction(name, types_of_arguments...) +# AggregateFunction(name, types_of_arguments...) Промежуточное состояние агрегатной функции. Чтобы его получить, используются агрегатные функции с суффиксом -State. Подробнее смотрите в разделе "AggregatingMergeTree". diff --git a/docs/ru/data_types/nested_data_structures/index.md b/docs/ru/data_types/nested_data_structures/index.md index 2d0ba1fcec8..9c926de6807 100644 --- a/docs/ru/data_types/nested_data_structures/index.md +++ b/docs/ru/data_types/nested_data_structures/index.md @@ -1,5 +1,4 @@ -Вложенные структуры данных --------------------------- +# Вложенные структуры данных ```eval_rst .. toctree:: diff --git a/docs/ru/data_types/nested_data_structures/nested.md b/docs/ru/data_types/nested_data_structures/nested.md index c5a3e6d7602..b513bc39938 100644 --- a/docs/ru/data_types/nested_data_structures/nested.md +++ b/docs/ru/data_types/nested_data_structures/nested.md @@ -1,4 +1,4 @@ -### Nested(Name1 Type1, Name2 Type2, ...) +# Nested(Name1 Type1, Name2 Type2, ...) Вложенная структура данных - это как будто вложенная таблица. Параметры вложенной структуры данных - имена и типы столбцов, указываются так же, как в запроса CREATE. Каждой строке таблицы может соответствовать произвольное количество строк вложенной структуры данных. diff --git a/docs/ru/data_types/special_data_types/expression.md b/docs/ru/data_types/special_data_types/expression.md index ee4ae7952c9..8451a034743 100644 --- a/docs/ru/data_types/special_data_types/expression.md +++ b/docs/ru/data_types/special_data_types/expression.md @@ -1,3 +1,3 @@ -### Expression +# Expression Используется для представления лямбда-выражений в функциях высшего порядка. diff --git a/docs/ru/data_types/special_data_types/index.md b/docs/ru/data_types/special_data_types/index.md index a894bd184e2..1bf83ad1af3 100644 --- a/docs/ru/data_types/special_data_types/index.md +++ b/docs/ru/data_types/special_data_types/index.md @@ -1,5 +1,4 @@ -Служебные типы данных ---------------------- +# Служебные типы данных Значения служебных типов данных не могут сохраняться в таблицу и выводиться в качестве результата, а возникают как промежуточный результат выполнения запроса. diff --git a/docs/ru/data_types/special_data_types/set.md b/docs/ru/data_types/special_data_types/set.md index de188e5c9f8..72a9a2647e5 100644 --- a/docs/ru/data_types/special_data_types/set.md +++ b/docs/ru/data_types/special_data_types/set.md @@ -1,3 +1,3 @@ -### Set +# Set Используется для представления правой части выражения IN. diff --git a/docs/ru/data_types/string.md b/docs/ru/data_types/string.md index fdb9890e7ba..e35fa6a892d 100644 --- a/docs/ru/data_types/string.md +++ b/docs/ru/data_types/string.md @@ -1,10 +1,9 @@ -String ------- +# String Строки произвольной длины. Длина не ограничена. Значение может содержать произвольный набор байт, включая нулевые байты. Таким образом, тип String заменяет типы VARCHAR, BLOB, CLOB и т. п. из других СУБД. -### Кодировки +## Кодировки В ClickHouse нет понятия кодировок. Строки могут содержать произвольный набор байт, который хранится и выводится, как есть. Если вам нужно хранить тексты, рекомендуется использовать кодировку UTF-8. По крайней мере, если у вас терминал работает в кодировке UTF-8 (это рекомендуется), вы сможете читать и писать свои значения без каких-либо преобразований. diff --git a/docs/ru/data_types/tuple.md b/docs/ru/data_types/tuple.md index 67fba16984a..abac42bc4b7 100644 --- a/docs/ru/data_types/tuple.md +++ b/docs/ru/data_types/tuple.md @@ -1,5 +1,4 @@ -Tuple(T1, T2, ...) ------------------- +# Tuple(T1, T2, ...) Кортежи не могут быть записаны в таблицы (кроме таблиц типа Memory). Они используется для временной группировки столбцов. Столбцы могут группироваться при использовании выражения IN в запросе, а также для указания нескольких формальных параметров лямбда-функций. Подробнее смотрите раздел "Операторы IN", "Функции высшего порядка". diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index c9136a01277..5258907f9d1 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -1,5 +1,4 @@ -Overview of ClickHouse architecture -=================================== +# Overview of ClickHouse architecture ClickHouse is a true column-oriented DBMS. Data is stored by columns, and during query execution data is processed by arrays (vectors or chunks of columns). Whenever possible, operations are dispatched on arrays, rather than on individual values. This is called "vectorized query execution," and it helps lower dispatch cost relative to the cost of actual data processing. @@ -7,29 +6,25 @@ ClickHouse is a true column-oriented DBMS. Data is stored by columns, and during > > There are two different approaches for speeding up query processing: vectorized query execution and runtime code generation. In the latter, the code is generated for every kind of query on the fly, removing all indirection and dynamic dispatch. Neither of these approaches is strictly better than the other. Runtime code generation can be better when it fuses many operations together, thus fully utilizing CPU execution units and the pipeline. Vectorized query execution can be less practical, because it involves temporary vectors that must be written to cache and read back. If the temporary data does not fit in the L2 cache, this becomes an issue. But vectorized query execution more easily utilizes the SIMD capabilities of the CPU. A [research paper](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) written by our friends shows that it is better to combine both approaches. ClickHouse mainly uses vectorized query execution and has limited initial support for runtime code generation (only the inner loop of the first stage of GROUP BY can be compiled). -Columns -------- +## Columns To represent columns in memory (actually, chunks of columns), the `IColumn` interface is used. This interface provides helper methods for implementation of various relational operators. Almost all operations are immutable: they do not modify the original column, but create a new modified one. For example, the `IColumn::filter` method accepts a filter byte mask and creates a new filtered column. It is used for the `WHERE` and `HAVING` relational operators. Additional examples: the `IColumn::permute` method to support `ORDER BY`, the `IColumn::cut` method to support `LIMIT`, and so on. Various `IColumn` implementations (`ColumnUInt8`, `ColumnString` and so on) are responsible for the memory layout of columns. Memory layout is usually a contiguous array. For the integer type of columns it is just one contiguous array, like `std::vector`. For `String` and `Array` columns, it is two vectors: one for all array elements, placed contiguously, and a second one for offsets to the beginning of each array. There is also `ColumnConst` that stores just one value in memory, but looks like a column. -Field ------ +## Field Nevertheless, it is possible to work with individual values as well. To represent an individual value, `Field` is used. `Field` is just a discriminated union of `UInt64`, `Int64`, `Float64`, `String` and `Array`. `IColumn` has the `operator[]` method to get the n-th value as a `Field`, and the `insert` method to append a `Field` to the end of a column. These methods are not very efficient, because they require dealing with temporary `Field` objects representing an individual value. There are more efficient methods, such as `insertFrom`, `insertRangeFrom`, and so on. `Field` doesn't have enough information about a specific data type for a table. For example, `UInt8`, `UInt16`, `UInt32`, and `UInt64` are all represented as `UInt64` in a `Field`. -Leaky abstractions ------------------- +## Leaky abstractions `IColumn` has methods for common relational transformations of data, but they don't meet all needs. For example, `ColumnUInt64` doesn't have a method to calculate the sum of two columns, and `ColumnString` doesn't have a method to run a substring search. These countless routines are implemented outside of `IColumn`. Various functions on columns can be implemented in a generic, non-efficient way using `IColumn` methods to extract `Field` values, or in a specialized way using knowledge of inner memory layout of data in a specific `IColumn` implementation. To do this, functions are cast to a specific `IColumn` type and deal with internal representation directly. For example, `ColumnUInt64` has the `getData` method that returns a reference to an internal array, then a separate routine reads or fills that array directly. In fact, we have "leaky abstractions" to allow efficient specializations of various routines. -Data types ----------- +## Data types `IDataType` is responsible for serialization and deserialization: for reading and writing chunks of columns or individual values in binary or text form. `IDataType` directly corresponds to data types in tables. For example, there are `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` and so on. @@ -40,8 +35,7 @@ Data types `IDataType` has helper methods for various data formats. Examples are methods to serialize a value with possible quoting, to serialize a value for JSON, and to serialize a value as part of XML format. There is no direct correspondence to data formats. For example, the different data formats `Pretty` and `TabSeparated` can use the same `serializeTextEscaped` helper method from the `IDataType` interface. -Block ------ +## Block A `Block` is a container that represents a subset (chunk) of a table in memory. It is just a set of triples: `(IColumn, IDataType, column name)`. During query execution, data is processed by `Block`s. If we have a `Block`, we have data (in the `IColumn` object), we have information about its type (in `IDataType`) that tells us how to deal with that column, and we have the column name (either the original column name from the table, or some artificial name assigned for getting temporary results of calculations). @@ -49,8 +43,7 @@ When we calculate some function over columns in a block, we add another column w Blocks are created for every processed chunk of data. Note that for the same type of calculation, the column names and types remain the same for different blocks, and only column data changes. It is better to split block data from the block header, because small block sizes will have a high overhead of temporary strings for copying shared_ptrs and column names. -Block Streams -------------- +## Block Streams Block streams are for processing data. We use streams of blocks to read data from somewhere, perform data transformations, or write data to somewhere. `IBlockInputStream` has the `read` method to fetch the next block while available. `IBlockOutputStream` has the `write` method to push the block somewhere. @@ -66,15 +59,13 @@ There are more sophisticated transformations. For example, when you pull from `A > > We should note that the query execution pipeline creates temporary data at each step. We try to keep block size small enough so that temporary data fits in the CPU cache. With that assumption, writing and reading temporary data is almost free in comparison with other calculations. We could consider an alternative, which is to fuse many operations in the pipeline together, to make the pipeline as short as possible and remove much of the temporary data. This could be an advantage, but it also has drawbacks. For example, a split pipeline makes it easy to implement caching intermediate data, stealing intermediate data from similar queries running at the same time, and merging pipelines for similar queries. -Formats -------- +## Formats Data formats are implemented with block streams. There are "presentational" formats only suitable for output of data to the client, such as `Pretty` format, which provides only `IBlockOutputStream`. And there are input/output formats, such as `TabSeparated` or `JSONEachRow`. There are also row streams: `IRowInputStream` and `IRowOutputStream`. They allow you to pull/push data by individual rows, not by blocks. And they are only needed to simplify implementation of row-oriented formats. The wrappers `BlockInputStreamFromRowInputStream` and `BlockOutputStreamFromRowOutputStream` allow you to convert row-oriented streams to regular block-oriented streams. -I/O ---- +## I/O For byte-oriented input/output, there are `ReadBuffer` and `WriteBuffer` abstract classes. They are used instead of C++ `iostream`'s. Don't worry: every mature C++ project is using something other than `iostream`'s for good reasons. @@ -86,8 +77,7 @@ Read/WriteBuffers only deal with bytes. To help with formatted input/output (for Let's look at what happens when you want to write a result set in `JSON` format to stdout. You have a result set ready to be fetched from `IBlockInputStream`. You create `WriteBufferFromFileDescriptor(STDOUT_FILENO)` to write bytes to stdout. You create `JSONRowOutputStream`, initialized with that `WriteBuffer`, to write rows in `JSON` to stdout. You create `BlockOutputStreamFromRowOutputStream` on top of it, to represent it as `IBlockOutputStream`. Then you call `copyData` to transfer data from `IBlockInputStream` to `IBlockOutputStream`, and everything works. Internally, `JSONRowOutputStream` will write various JSON delimiters and call the `IDataType::serializeTextJSON` method with a reference to `IColumn` and the row number as arguments. Consequently, `IDataType::serializeTextJSON` will call a method from `WriteHelpers.h`: for example, `writeText` for numeric types and `writeJSONString` for `DataTypeString`. -Tables ------- +## Tables Tables are represented by the `IStorage` interface. Different implementations of that interface are different table engines. Examples are `StorageMergeTree`, `StorageMemory`, and so on. Instances of these classes are just tables. @@ -108,22 +98,19 @@ To get a quick idea of how to implement your own table engine, look at something > As the result of the `read` method, `IStorage` returns `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. Currently we have only very coarse granularity for that information. There is no way for the storage to say "I have already processed this part of the expression in WHERE, for this range of data". We need to work on that. -Parsers -------- +## Parsers A query is parsed by a hand-written recursive descent parser. For example, `ParserSelectQuery` just recursively calls the underlying parsers for various parts of the query. Parsers create an `AST`. The `AST` is represented by nodes, which are instances of `IAST`. > Parser generators are not used for historical reasons. -Interpreters ------------- +## Interpreters Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery`and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to; and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time. `InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations or query. -Functions ---------- +## Functions There are ordinary functions and aggregate functions. For aggregate functions, see the next section. @@ -139,8 +126,7 @@ Implementing a function may be slightly inconvenient because a function explicit > > Due to vectorized query execution, functions are not short-circuit. For example, if you write `WHERE f(x) AND g(y)`, both sides will be calculated, even for rows, when `f(x)` is zero (except when `f(x)` is a zero constant expression). But if selectivity of the `f(x)` condition is high, and calculation of `f(x)` is much cheaper than `g(y)`, it's better to implement multi-pass calculation: first calculate `f(x)`, then filter columns by the result, and then calculate `g(y)` only for smaller, filtered chunks of data. -Aggregate Functions -------------------- +## Aggregate Functions Aggregate functions are stateful functions. They accumulate passed values into some state, and allow you to get results from that state. They are managed with the `IAggregateFunction` interface. States can be rather simple (the state for `AggregateFunctionCount` is just a single `UInt64` value) or quite complex (the state of `AggregateFunctionUniqCombined` is a combination of a linear array, a hash table and a `HyperLogLog` probabilistic data structure). @@ -150,8 +136,7 @@ Aggregation states can be serialized and deserialized to pass over the network d > The serialized data format for aggregate function states is not versioned right now. This is ok if aggregate states are only stored temporarily. But we have the `AggregatingMergeTree` table engine for incremental aggregation, and people are already using it in production. This is why we should add support for backward compatibility when changing the serialized format for any aggregate function in the future. -Server ------- +## Server The server implements several different interfaces: - An HTTP interface for any foreign clients. @@ -166,8 +151,7 @@ We maintain full backward and forward compatibility for the server TCP protocol: > For all external applications, we recommend using the HTTP interface because it is simple and easy to use. The TCP protocol is more tightly linked to internal data structures: it uses an internal format for passing blocks of data and it uses custom framing for compressed data. We haven't released a C library for that protocol because it requires linking most of the ClickHouse codebase, which is not practical. -Distributed query execution ---------------------------- +## Distributed query execution Servers in a cluster setup are mostly independent. You can create a `Distributed` table on one or all servers in a cluster. The `Distributed` table does not store data itself – it only provides a "view" to all local tables on multiple nodes of a cluster. When you SELECT from a `Distributed` table, it rewrites that query, chooses remote nodes according to load balancing settings, and sends the query to them. The `Distributed` table requests remote servers to process a query just up to a stage where intermediate results from different servers can be merged. Then it receives the intermediate results and merges them. The distributed table tries to distribute as much work as possible to remote servers, and does not send much intermediate data over the network. @@ -175,8 +159,7 @@ Servers in a cluster setup are mostly independent. You can create a `Distributed > > There is no global query plan for distributed query execution. Each node has its own local query plan for its part of the job. We only have simple one-pass distributed query execution: we send queries for remote nodes and then merge the results. But this is not feasible for difficult queries with high cardinality GROUP BYs or with a large amount of temporary data for JOIN: in such cases, we need to "reshuffle" data between servers, which requires additional coordination. ClickHouse does not support that kind of query execution, and we need to work on it. -Merge Tree ----------- +## Merge Tree `MergeTree` is a family of storage engines that supports indexing by primary key. The primary key can be an arbitary tuple of columns or expressions. Data in a `MergeTree` table is stored in "parts". Each part stores data in the primary key order (data is ordered lexicographically by the primary key tuple). All the table columns are stored in separate `column.bin` files in these parts. The files consist of compressed blocks. Each block is usually from 64 KB to 1 MB of uncompressed data, depending on the average value size. The blocks consist of column values placed contiguously one after the other. Column values are in the same order for each column (the order is defined by the primary key), so when you iterate by many columns, you get values for the corresponding rows. @@ -192,8 +175,7 @@ When you `INSERT` a bunch of data into `MergeTree`, that bunch is sorted by prim > > There are MergeTree engines that are doing additional work during background merges. Examples are `CollapsingMergeTree` and `AggregatingMergeTree`. This could be treated as special support for updates. Keep in mind that these are not real updates because users usually have no control over the time when background merges will be executed, and data in a `MergeTree` table is almost always stored in more than one part, not in completely merged form. -Replication ------------ +## Replication Replication in ClickHouse is implemented on a per-table basis. You could have some replicated and some non-replicated tables on the same server. You could also have tables replicated in different ways, such as one table with two-factor replication and another with three-factor. diff --git a/docs/ru/development/build.md b/docs/ru/development/build.md index 4e12fab3621..673192a14f7 100644 --- a/docs/ru/development/build.md +++ b/docs/ru/development/build.md @@ -1,5 +1,4 @@ -How to build ClickHouse on Linux -================================ +# How to build ClickHouse on Linux Build should work on Linux Ubuntu 12.04, 14.04 or newer. With appropriate changes, build should work on any other Linux distribution. @@ -12,8 +11,7 @@ To test for SSE 4.2, do grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" ``` -Install Git and CMake ---------------------- +## Install Git and CMake ```bash sudo apt-get install git cmake3 @@ -21,15 +19,13 @@ sudo apt-get install git cmake3 Or just cmake on newer systems. -Detect number of threads ------------------------- +## Detect number of threads ```bash export THREADS=$(grep -c ^processor /proc/cpuinfo) ``` -Install GCC 7 -------------- +## Install GCC 7 There are several ways to do it. @@ -46,23 +42,20 @@ sudo apt-get install gcc-7 g++-7 Look at [https://github.com/yandex/ClickHouse/blob/master/utils/prepare-environment/install-gcc.sh] -Use GCC 7 for builds --------------------- +## Use GCC 7 for builds ```bash export CC=gcc-7 export CXX=g++-7 ``` -Install required libraries from packages ----------------------------------------- +## Install required libraries from packages ```bash sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev ``` -Checkout ClickHouse sources ---------------------------- +## Checkout ClickHouse sources To get latest stable version: @@ -76,8 +69,7 @@ cd ClickHouse For development, switch to the `master` branch. For latest release candidate, switch to the `testing` branch. -Build ClickHouse ----------------- +## Build ClickHouse There are two variants of build. diff --git a/docs/ru/development/build_osx.md b/docs/ru/development/build_osx.md index c13f8ce6413..b4ae2ba0453 100644 --- a/docs/ru/development/build_osx.md +++ b/docs/ru/development/build_osx.md @@ -1,25 +1,21 @@ -How to build ClickHouse on Mac OS X -=================================== +# How to build ClickHouse on Mac OS X Build should work on Mac OS X 10.12. If you're using earlier version, you can try to build ClickHouse using Gentoo Prefix and clang sl in this instruction. With appropriate changes, build should work on any other OS X distribution. -Install Homebrew ----------------- +## Install Homebrew ```bash /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -Install required compilers, tools, libraries --------------------------------------------- +## Install required compilers, tools, libraries ```bash brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext homebrew/dupes/zlib readline boost --cc=gcc-7 ``` -Checkout ClickHouse sources ---------------------------- +## Checkout ClickHouse sources To get the latest stable version: @@ -33,8 +29,7 @@ cd ClickHouse For development, switch to the `master` branch. For the latest release candidate, switch to the `testing` branch. -Build ClickHouse ----------------- +## Build ClickHouse ```bash mkdir build @@ -44,7 +39,6 @@ make -j `sysctl -n hw.ncpu` cd .. ``` -Caveats -------- +## Caveats If you intend to run clickhouse-server, make sure to increase system's maxfiles variable. See [MacOS.md](https://github.com/yandex/ClickHouse/blob/master/MacOS.md) for more details. diff --git a/docs/ru/development/index.md b/docs/ru/development/index.md index eab34ba26e3..f2b123c8371 100644 --- a/docs/ru/development/index.md +++ b/docs/ru/development/index.md @@ -1,5 +1,4 @@ -ClickHouse Development -====================== +# ClickHouse Development ```eval_rst .. toctree:: diff --git a/docs/ru/development/tests.md b/docs/ru/development/tests.md index a3febcea92d..c7f90cdcd45 100644 --- a/docs/ru/development/tests.md +++ b/docs/ru/development/tests.md @@ -1,5 +1,4 @@ -How to run ClickHouse tests -=========================== +# How to run ClickHouse tests The `clickhouse-test` utility that is used for functional testing is written using Python 2.x. It also requires you to have some third-party packages: @@ -16,8 +15,7 @@ In a nutshell: - `cd dbms/tests/` - Run `./clickhouse-test` -Example usage -------------- +## Example usage Run `./clickhouse-test --help` to see available options. diff --git a/docs/ru/dicts/external_dicts.md b/docs/ru/dicts/external_dicts.md index ec15c75eb73..e77d5e6b841 100644 --- a/docs/ru/dicts/external_dicts.md +++ b/docs/ru/dicts/external_dicts.md @@ -1,7 +1,6 @@ -Внешние словари -=============== +# Внешние словари Существует возможность подключать собственные словари из различных источников данных. Источником данных для словаря может быть локальный текстовый/исполняемый файл, HTTP(s) ресурс или другая СУБД. Подробнее смотрите в разделе "[Источники внешних словарей](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources)". diff --git a/docs/ru/dicts/external_dicts_dict.md b/docs/ru/dicts/external_dicts_dict.md index 584bffb7915..ecab0af8e82 100644 --- a/docs/ru/dicts/external_dicts_dict.md +++ b/docs/ru/dicts/external_dicts_dict.md @@ -1,7 +1,6 @@ -Настройка внешнего словаря -========================== +# Настройка внешнего словаря Конфигурация словаря имеет следующую структуру: diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md index b3dc7bd41e0..427a99f057f 100644 --- a/docs/ru/dicts/external_dicts_dict_layout.md +++ b/docs/ru/dicts/external_dicts_dict_layout.md @@ -1,7 +1,6 @@ -Хранение словарей в памяти -========================== +# Хранение словарей в памяти Словари можно размещать в памяти [множеством способов](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner). @@ -39,8 +38,7 @@ -Способы размещения словарей в памяти -==================================== +## Способы размещения словарей в памяти - [flat](#dicts-external_dicts_dict_layout-flat) - [hashed](#dicts-external_dicts_dict_layout-hashed) @@ -51,8 +49,7 @@ -flat ----- +### flat Словарь полностью хранится в оперативной памяти в виде плоских массивов. Объем памяти, занимаемой словарем? пропорционален размеру самого большого (по размеру) ключа. @@ -72,8 +69,7 @@ flat -hashed ------- +### hashed Словарь полностью хранится в оперативной памяти в виде хэш-таблиц. Словарь может содержать произвольное количество элементов с произвольными идентификаторами. На практике, количество ключей может достигать десятков миллионов элементов. @@ -89,8 +85,7 @@ hashed -complex_key_hashed ------------------- +### complex_key_hashed Тип размещения предназначен для использования с составными [ключами](..external_dicts_dict_structure.html/#dicts-external_dicts_dict_structure). Аналогичен hashed. @@ -104,8 +99,7 @@ complex_key_hashed -range_hashed ------------- +### range_hashed Словарь хранится в оперативной памяти в виде хэш-таблицы с упорядоченным массивом диапазонов и соответствующих им значений. @@ -190,8 +184,7 @@ range_hashed -cache ------ +### cache Словарь хранится в кэше, состоящем из фиксированного количества ячеек. Ячейки содержат часто используемые элементы. @@ -231,7 +224,6 @@ cache -complex_key_cache ------------------ +### complex_key_cache Тип размещения предназначен для использования с составными [ключами](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Аналогичен `cache`. diff --git a/docs/ru/dicts/external_dicts_dict_lifetime.md b/docs/ru/dicts/external_dicts_dict_lifetime.md index fcfbe011489..4cd4238125c 100644 --- a/docs/ru/dicts/external_dicts_dict_lifetime.md +++ b/docs/ru/dicts/external_dicts_dict_lifetime.md @@ -1,7 +1,6 @@ -Обновление словарей -=================== +# Обновление словарей ClickHouse периодически обновляет словари. Интервал обновления для полностью загружаемых словарей и интервал инвалидации для кэшируемых словарей определяется в теге `` в секундах. diff --git a/docs/ru/dicts/external_dicts_dict_sources.md b/docs/ru/dicts/external_dicts_dict_sources.md index b29e2546814..ce5c06202f5 100644 --- a/docs/ru/dicts/external_dicts_dict_sources.md +++ b/docs/ru/dicts/external_dicts_dict_sources.md @@ -1,7 +1,6 @@ -Источники внешних словарей -========================== +# Источники внешних словарей Внешний словарь можно подключить из множества источников. @@ -37,8 +36,7 @@ -Локальный файл --------------- +## Локальный файл Пример настройки: @@ -58,8 +56,7 @@ -Исполняемый файл ----------------- +## Исполняемый файл Работа с исполняемым файлом зависит от [размещения словаря в памяти](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). Если тип размещения словаря `cache` и `complex_key_cache`, то ClickHouse запрашивает необходимые ключи, отправляя запрос в `STDIN` исполняемого файла. @@ -81,8 +78,7 @@ -HTTP(s) -------- +## HTTP(s) Работа с HTTP(s) сервером зависит от [размещения словаря в памяти](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). Если тип размещения словаря `cache` и `complex_key_cache`, то ClickHouse запрашивает необходимые ключи, отправляя запрос методом `POST`. @@ -106,8 +102,7 @@ HTTP(s) -ODBC ----- +## ODBC Этим способом можно подключить любую базу данных, имеющую ODBC драйвер. @@ -129,8 +124,7 @@ ODBC - `connection_string` - строка соединения. - `invalidate_query` - запрос для проверки статуса словаря. Необязательный параметр. Читайте подробнее в разделе [Обновление словарей](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime). -Пример подключения PostgreSQL ------------------------------ +## Пример подключения PostgreSQL ОС Ubuntu. @@ -273,8 +267,7 @@ ODBC ``` -СУБД ----- +## СУБД diff --git a/docs/ru/dicts/external_dicts_dict_structure.md b/docs/ru/dicts/external_dicts_dict_structure.md index 38b7063dce0..99792b8c465 100644 --- a/docs/ru/dicts/external_dicts_dict_structure.md +++ b/docs/ru/dicts/external_dicts_dict_structure.md @@ -1,7 +1,6 @@ -Ключ и поля словаря -=================== +# Ключ и поля словаря Секция `` описывает ключ словаря и поля, доступные для запросов. @@ -31,8 +30,7 @@ -Ключ ----- +## Ключ ClickHouse поддерживает следующие виды ключей: @@ -95,8 +93,7 @@ Cоставной ключ может состоять и из одного эл -Атрибуты --------- +## Атрибуты Пример конфигурации: diff --git a/docs/ru/dicts/index.md b/docs/ru/dicts/index.md index 12e23d3f19d..98bb78f0424 100644 --- a/docs/ru/dicts/index.md +++ b/docs/ru/dicts/index.md @@ -1,5 +1,4 @@ -Словари -======= +# Словари `Словарь` - это отображение (ключ `->` атрибуты), которое можно использовать в запросе в виде функций. Это можно рассматривать как более удобный и максимально эффективный вариант JOIN-а с таблицами-справочниками (dimension tables). diff --git a/docs/ru/dicts/internal_dicts.md b/docs/ru/dicts/internal_dicts.md index 582f2c0fa8b..6b765c6f55f 100644 --- a/docs/ru/dicts/internal_dicts.md +++ b/docs/ru/dicts/internal_dicts.md @@ -1,5 +1,4 @@ -Встроенные словари -================== +# Встроенные словари ClickHouse содержит встроенную возможность работы с геобазой. diff --git a/docs/ru/formats/capnproto.md b/docs/ru/formats/capnproto.md index 64043fe7889..3364b1affc1 100644 --- a/docs/ru/formats/capnproto.md +++ b/docs/ru/formats/capnproto.md @@ -1,7 +1,6 @@ -CapnProto -========= +# CapnProto Cap'n Proto - формат бинарных сообщений, похож на Protocol Buffers и Thrift, но не похож на JSON или MessagePack. diff --git a/docs/ru/formats/csv.md b/docs/ru/formats/csv.md index d2c70644897..84c5cc08cb3 100644 --- a/docs/ru/formats/csv.md +++ b/docs/ru/formats/csv.md @@ -1,5 +1,4 @@ -CSV -=== +# CSV Формат comma separated values ([RFC](https://tools.ietf.org/html/rfc4180)). diff --git a/docs/ru/formats/csvwithnames.md b/docs/ru/formats/csvwithnames.md index dedfb83d63f..a9f08b826db 100644 --- a/docs/ru/formats/csvwithnames.md +++ b/docs/ru/formats/csvwithnames.md @@ -1,4 +1,3 @@ -CSVWithNames -============ +# CSVWithNames Выводит также заголовок, аналогично `TabSeparatedWithNames`. diff --git a/docs/ru/formats/index.md b/docs/ru/formats/index.md index f9b6979a69b..d145ff5dd82 100644 --- a/docs/ru/formats/index.md +++ b/docs/ru/formats/index.md @@ -1,7 +1,6 @@ -Форматы -======= +# Форматы Формат определяет, в каком виде данные отдаются вам (пишутся, форматируются сервером) при SELECT-е и в каком виде принимаются (читаются, парсятся сервером) при INSERT-е. diff --git a/docs/ru/formats/json.md b/docs/ru/formats/json.md index 81a30ad6adc..00d26d9e597 100644 --- a/docs/ru/formats/json.md +++ b/docs/ru/formats/json.md @@ -1,5 +1,4 @@ -JSON -==== +# JSON Выводит данные в формате JSON. Кроме таблицы с данными, также выводятся имена и типы столбцов, и некоторая дополнительная информация - общее количество выведенных строк, а также количество строк, которое могло бы быть выведено, если бы не было LIMIT-а. Пример: diff --git a/docs/ru/formats/jsoncompact.md b/docs/ru/formats/jsoncompact.md index d9bf3623a75..10bbd530ffa 100644 --- a/docs/ru/formats/jsoncompact.md +++ b/docs/ru/formats/jsoncompact.md @@ -1,5 +1,4 @@ -JSONCompact -=========== +# JSONCompact Отличается от JSON только тем, что строчки данных выводятся в массивах, а не в object-ах. diff --git a/docs/ru/formats/jsoneachrow.md b/docs/ru/formats/jsoneachrow.md index 7a5166f1b95..6efd15936ff 100644 --- a/docs/ru/formats/jsoneachrow.md +++ b/docs/ru/formats/jsoneachrow.md @@ -1,5 +1,4 @@ -JSONEachRow -=========== +# JSONEachRow Выводит данные в виде отдельных JSON объектов для каждой строки (newline delimited JSON). diff --git a/docs/ru/formats/native.md b/docs/ru/formats/native.md index c0373823bc2..e7aa5b323c4 100644 --- a/docs/ru/formats/native.md +++ b/docs/ru/formats/native.md @@ -1,5 +1,4 @@ -Native -====== +# Native Самый эффективный формат. Данные пишутся и читаются блоками в бинарном виде. Для каждого блока пишется количество строк, количество столбцов, имена и типы столбцов, а затем кусочки столбцов этого блока, один за другим. То есть, этот формат является "столбцовым" - не преобразует столбцы в строки. Именно этот формат используется в родном интерфейсе - при межсерверном взаимодействии, при использовании клиента командной строки, при работе клиентов, написанных на C++. diff --git a/docs/ru/formats/null.md b/docs/ru/formats/null.md index 6b9bc1ff674..ac699e493a7 100644 --- a/docs/ru/formats/null.md +++ b/docs/ru/formats/null.md @@ -1,5 +1,4 @@ -Null -==== +# Null Ничего не выводит. При этом, запрос обрабатывается, а при использовании клиента командной строки, данные ещё и передаются на клиент. Используется для тестов, в том числе, тестов производительности. Очевидно, формат подходит только для вывода, но не для парсинга. diff --git a/docs/ru/formats/pretty.md b/docs/ru/formats/pretty.md index 8ac60e780f7..cac5b7ed1da 100644 --- a/docs/ru/formats/pretty.md +++ b/docs/ru/formats/pretty.md @@ -1,5 +1,4 @@ -Pretty -====== +# Pretty Выводит данные в виде Unicode-art табличек, также используя ANSI-escape последовательности для установки цветов в терминале. Рисуется полная сетка таблицы и, таким образом, каждая строчка занимает две строки в терминале. diff --git a/docs/ru/formats/prettycompact.md b/docs/ru/formats/prettycompact.md index f9bf520a2bb..5802dfbc1ef 100644 --- a/docs/ru/formats/prettycompact.md +++ b/docs/ru/formats/prettycompact.md @@ -1,5 +1,4 @@ -PrettyCompact -============= +# PrettyCompact Отличается от `Pretty` тем, что не рисуется сетка между строками - результат более компактный. Этот формат используется по умолчанию в клиенте командной строки в интерактивном режиме. diff --git a/docs/ru/formats/prettycompactmonoblock.md b/docs/ru/formats/prettycompactmonoblock.md index 54d51c7238b..3ac6b4fcd76 100644 --- a/docs/ru/formats/prettycompactmonoblock.md +++ b/docs/ru/formats/prettycompactmonoblock.md @@ -1,4 +1,3 @@ -PrettyCompactMonoBlock -====================== +# PrettyCompactMonoBlock Отличается от `PrettyCompact` тем, что строки (до 10 000 штук) буферизуются и затем выводятся в виде одной таблицы, а не по блокам. diff --git a/docs/ru/formats/prettynoescapes.md b/docs/ru/formats/prettynoescapes.md index 112f6dea1ea..b33eebe2907 100644 --- a/docs/ru/formats/prettynoescapes.md +++ b/docs/ru/formats/prettynoescapes.md @@ -1,5 +1,4 @@ -PrettyNoEscapes -=============== +# PrettyNoEscapes Отличается от Pretty тем, что не используются ANSI-escape последовательности. Это нужно для отображения этого формата в браузере, а также при использовании утилиты командной строки watch. @@ -11,12 +10,10 @@ watch -n1 "clickhouse-client --query='SELECT * FROM system.events FORMAT PrettyC Для отображения в браузере, вы можете использовать HTTP интерфейс. -PrettyCompactNoEscapes -====================== +## PrettyCompactNoEscapes Аналогично. -PrettySpaceNoEscapes -==================== +## PrettySpaceNoEscapes Аналогично. diff --git a/docs/ru/formats/prettyspace.md b/docs/ru/formats/prettyspace.md index e564c98eefe..10ba36f6182 100644 --- a/docs/ru/formats/prettyspace.md +++ b/docs/ru/formats/prettyspace.md @@ -1,4 +1,3 @@ -PrettySpace -=========== +# PrettySpace Отличается от `PrettyCompact` тем, что вместо сетки используется пустое пространство (пробелы). diff --git a/docs/ru/formats/rowbinary.md b/docs/ru/formats/rowbinary.md index 95118bf5fa5..24b3c5c5005 100644 --- a/docs/ru/formats/rowbinary.md +++ b/docs/ru/formats/rowbinary.md @@ -1,5 +1,4 @@ -RowBinary -========= +# RowBinary Форматирует и парсит данные по строкам, в бинарном виде. Строки и значения уложены подряд, без разделителей. Формат менее эффективен, чем формат Native, так как является строковым. diff --git a/docs/ru/formats/tabseparated.md b/docs/ru/formats/tabseparated.md index a508ce46cfd..4a2c7ea9abf 100644 --- a/docs/ru/formats/tabseparated.md +++ b/docs/ru/formats/tabseparated.md @@ -1,5 +1,4 @@ -TabSeparated -============ +# TabSeparated В TabSeparated формате данные пишутся по строкам. Каждая строчка содержит значения, разделённые табами. После каждого значения идёт таб, кроме последнего значения в строке, после которого идёт перевод строки. Везде подразумеваются исключительно unix-переводы строк. Последняя строка также обязана содержать перевод строки на конце. Значения пишутся в текстовом виде, без обрамляющих кавычек, с экранированием служебных символов. diff --git a/docs/ru/formats/tabseparatedraw.md b/docs/ru/formats/tabseparatedraw.md index eb761ece98a..f05f5b64f01 100644 --- a/docs/ru/formats/tabseparatedraw.md +++ b/docs/ru/formats/tabseparatedraw.md @@ -1,5 +1,4 @@ -TabSeparatedRaw -=============== +# TabSeparatedRaw Отличается от формата `TabSeparated` тем, что строки выводятся без экранирования. Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). diff --git a/docs/ru/formats/tabseparatedwithnames.md b/docs/ru/formats/tabseparatedwithnames.md index bab1818c97e..d69fef92d46 100644 --- a/docs/ru/formats/tabseparatedwithnames.md +++ b/docs/ru/formats/tabseparatedwithnames.md @@ -1,5 +1,4 @@ -TabSeparatedWithNames -===================== +# TabSeparatedWithNames Отличается от формата `TabSeparated` тем, что в первой строке пишутся имена столбцов. При парсинге, первая строка полностью игнорируется: вы не можете использовать имена столбцов, чтобы указать их порядок расположения, или чтобы проверить их корректность. diff --git a/docs/ru/formats/tabseparatedwithnamesandtypes.md b/docs/ru/formats/tabseparatedwithnamesandtypes.md index d033c4ef58e..8e024e28259 100644 --- a/docs/ru/formats/tabseparatedwithnamesandtypes.md +++ b/docs/ru/formats/tabseparatedwithnamesandtypes.md @@ -1,5 +1,4 @@ -TabSeparatedWithNamesAndTypes -============================= +# TabSeparatedWithNamesAndTypes Отличается от формата `TabSeparated` тем, что в первой строке пишутся имена столбцов, а во второй - типы столбцов. При парсинге, первая и вторая строка полностью игнорируется. diff --git a/docs/ru/formats/tskv.md b/docs/ru/formats/tskv.md index 5eedcfe677e..0f61cab26e8 100644 --- a/docs/ru/formats/tskv.md +++ b/docs/ru/formats/tskv.md @@ -1,5 +1,4 @@ -TSKV -==== +# TSKV Похож на TabSeparated, но выводит значения в формате name=value. Имена экранируются так же, как строки в формате TabSeparated и, дополнительно, экранируется также символ =. diff --git a/docs/ru/formats/values.md b/docs/ru/formats/values.md index 5449b2e98f4..076c6195b16 100644 --- a/docs/ru/formats/values.md +++ b/docs/ru/formats/values.md @@ -1,5 +1,4 @@ -Values -====== +# Values Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату TabSeparated. При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). diff --git a/docs/ru/formats/vertical.md b/docs/ru/formats/vertical.md index 17bccfa53c2..0496defb293 100644 --- a/docs/ru/formats/vertical.md +++ b/docs/ru/formats/vertical.md @@ -1,5 +1,4 @@ -Vertical -======== +# Vertical Выводит каждое значение на отдельной строке, с указанием имени столбца. Формат удобно использовать для вывода одной-нескольких строк, если каждая строка состоит из большого количества столбцов. Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). diff --git a/docs/ru/formats/xml.md b/docs/ru/formats/xml.md index cd8dfad9b7c..66535cf7d02 100644 --- a/docs/ru/formats/xml.md +++ b/docs/ru/formats/xml.md @@ -1,5 +1,4 @@ -XML -=== +# XML Формат XML подходит только для вывода данных, не для парсинга. Пример: diff --git a/docs/ru/functions/arithmetic_functions.md b/docs/ru/functions/arithmetic_functions.md index 2bc3b89af72..70c6826d8b3 100644 --- a/docs/ru/functions/arithmetic_functions.md +++ b/docs/ru/functions/arithmetic_functions.md @@ -1,5 +1,4 @@ -Арифметические функции -====================== +# Арифметические функции Для всех арифметических функций, тип результата вычисляется, как минимальный числовой тип, который может вместить результат, если такой тип есть. Минимум берётся одновременно по числу бит, знаковости и "плавучести". Если бит не хватает, то берётся тип максимальной битности. @@ -19,67 +18,56 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 Переполнение производится также, как в C++. -plus(a, b), оператор a + b --------------------------- +## plus(a, b), оператор a + b Вычисляет сумму чисел. Также можно складывать целые числа с датой и датой-с-временем. В случае даты, прибавление целого числа означает прибавление соответствующего количества дней. В случае даты-с-временем - прибавление соответствующего количества секунд. -minus(a, b), оператор a - b ---------------------------- +## minus(a, b), оператор a - b Вычисляет разность чисел. Результат всегда имеет знаковый тип. Также можно вычитать целые числа из даты и даты-с-временем. Смысл аналогичен - смотрите выше для plus. -multiply(a, b), оператор a \* b -------------------------------- +## multiply(a, b), оператор a \* b Вычисляет произведение чисел. -divide(a, b), оператор a / b ----------------------------- +## divide(a, b), оператор a / b Вычисляет частное чисел. Тип результата всегда является типом с плавающей запятой. То есть, деление не целочисленное. Для целочисленного деления, используйте функцию intDiv. При делении на ноль получится inf, -inf или nan. -intDiv(a, b) ------------- +## intDiv(a, b) Вычисляет частное чисел. Деление целочисленное, с округлением вниз (по абсолютному значению). При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. -intDivOrZero(a, b) ------------------- +## intDivOrZero(a, b) Отличается от intDiv тем, что при делении на ноль или при делении минимального отрицательного числа на минус единицу, возвращается ноль. -modulo(a, b), оператор a % b ----------------------------- +## modulo(a, b), оператор a % b Вычисляет остаток от деления. Если аргументы - числа с плавающей запятой, то они предварительно преобразуются в целые числа, путём отбрасывания дробной части. Берётся остаток в том же смысле, как это делается в C++. По факту, для отрицательных чисел, используется truncated division. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. -negate(a), оператор -a ----------------------- +## negate(a), оператор -a Вычисляет число, обратное по знаку. Результат всегда имеет знаковый тип. -abs(a) ------- +## abs(a) Вычисляет абсолютное значение для числа a. То есть, если a < 0, то возвращает -a. Для беззнаковых типов ничего не делает. Для чисел типа целых со знаком, возвращает число беззнакового типа. -gcd(a, b) ---------- +## gcd(a, b) Вычисляет наибольший общий делитель чисел. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. -lcm(a, b) ---------- +## lcm(a, b) Вычисляет наименьшее общее кратное чисел. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. diff --git a/docs/ru/functions/array_join.md b/docs/ru/functions/array_join.md index 25e63379ef3..40878eee027 100644 --- a/docs/ru/functions/array_join.md +++ b/docs/ru/functions/array_join.md @@ -1,7 +1,6 @@ -Функция arrayJoin -================= +# Функция arrayJoin Это совсем необычная функция. diff --git a/docs/ru/functions/bit_functions.md b/docs/ru/functions/bit_functions.md index 0ebdd894ca2..b2fc83b6e0f 100644 --- a/docs/ru/functions/bit_functions.md +++ b/docs/ru/functions/bit_functions.md @@ -1,18 +1,17 @@ -Битовые функции -=============== +# Битовые функции Битовые функции работают для любой пары типов из UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64. Тип результата - целое число, битность которого равна максимальной битности аргументов. Если хотя бы один аргумент знаковый, то результат - знаковое число. Если аргумент - число с плавающей запятой - оно приводится к Int64. -### bitAnd(a, b) +## bitAnd(a, b) -### bitOr(a, b) +## bitOr(a, b) -### bitXor(a, b) +## bitXor(a, b) -### bitNot(a) +## bitNot(a) -### bitShiftLeft(a, b) +## bitShiftLeft(a, b) -### bitShiftRight(a, b) +## bitShiftRight(a, b) diff --git a/docs/ru/functions/comparison_functions.md b/docs/ru/functions/comparison_functions.md index 06ede4232d6..99879e2dc7d 100644 --- a/docs/ru/functions/comparison_functions.md +++ b/docs/ru/functions/comparison_functions.md @@ -1,5 +1,5 @@ -Функции сравнения -================= +# Функции сравнения + Функции сравнения возвращают всегда 0 или 1 (UInt8). @@ -18,14 +18,14 @@ Замечание. До версии 1.1.54134 сравнение знаковых и беззнаковых целых чисел производилось также, как в C++. То есть, вы могли получить неверный результат в таких случаях: SELECT 9223372036854775807 > -1. С версии 1.1.54134 поведение изменилось и стало математически корректным. -### equals, оператор a = b и a == b +## equals, оператор a = b и a == b -### notEquals, оператор a != b и a `<>` b +## notEquals, оператор a != b и a `<>` b -### less, оператор `<` +## less, оператор `<` -### greater, оператор `>` +## greater, оператор `>` -### lessOrEquals, оператор `<=` +## lessOrEquals, оператор `<=` -### greaterOrEquals, оператор `>=` +## greaterOrEquals, оператор `>=` diff --git a/docs/ru/functions/conditional_functions.md b/docs/ru/functions/conditional_functions.md index 569463a9332..f7c58807138 100644 --- a/docs/ru/functions/conditional_functions.md +++ b/docs/ru/functions/conditional_functions.md @@ -1,6 +1,5 @@ -Условные функции -================ +# Условные функции -### if(cond, then, else), оператор cond ? then : else +## if(cond, then, else), оператор cond ? then : else Возвращает then, если cond != 0 или else, если cond = 0. cond должно иметь тип UInt8, а then и else должны иметь тип, для которого есть наименьший общий тип. diff --git a/docs/ru/functions/date_time_functions.md b/docs/ru/functions/date_time_functions.md index ad5d6080136..6f5d017cee8 100644 --- a/docs/ru/functions/date_time_functions.md +++ b/docs/ru/functions/date_time_functions.md @@ -1,5 +1,4 @@ -Функции для работы с датами и временем -====================================== +# Функции для работы с датами и временем Поддержка часовых поясов @@ -21,99 +20,99 @@ SELECT Поддерживаются только часовые пояса, отличающиеся от UTC на целое число часов. -### toYear +## toYear Переводит дату или дату-с-временем в число типа UInt16, содержащее номер года (AD). -### toMonth +## toMonth Переводит дату или дату-с-временем в число типа UInt8, содержащее номер месяца (1-12). -### toDayOfMonth +## toDayOfMonth Переводит дату или дату-с-временем в число типа UInt8, содержащее номер дня в месяце (1-31). -### toDayOfWeek +## toDayOfWeek Переводит дату или дату-с-временем в число типа UInt8, содержащее номер дня в неделе (понедельник - 1, воскресенье - 7). -### toHour +## toHour Переводит дату-с-временем в число типа UInt8, содержащее номер часа в сутках (0-23). Функция исходит из допущения, что перевод стрелок вперёд, если осуществляется, то на час, в два часа ночи, а перевод стрелок назад, если осуществляется, то на час, в три часа ночи (что, в общем, не верно - даже в Москве два раза перевод стрелок был осуществлён в другое время). -### toMinute +## toMinute Переводит дату-с-временем в число типа UInt8, содержащее номер минуты в часе (0-59). -### toSecond +## toSecond Переводит дату-с-временем в число типа UInt8, содержащее номер секунды в минуте (0-59). Секунды координации не учитываются. -### toMonday +## toMonday Округляет дату или дату-с-временем вниз до ближайшего понедельника. Возвращается дата. -### toStartOfMonth +## toStartOfMonth Округляет дату или дату-с-временем вниз до первого дня месяца. Возвращается дата. -### toStartOfQuarter +## toStartOfQuarter Округляет дату или дату-с-временем вниз до первого дня квартала. Первый день квартала - это одно из 1 января, 1 апреля, 1 июля, 1 октября. Возвращается дата. -### toStartOfYear +## toStartOfYear Округляет дату или дату-с-временем вниз до первого дня года. Возвращается дата. -### toStartOfMinute +## toStartOfMinute Округляет дату-с-временем вниз до начала минуты. -### toStartOfFiveMinute +## toStartOfFiveMinute Округляет дату-с-временем вниз до начала пятиминутного интервала. Замечание: если вам нужно округлить дату-с-временем до какого-либо другого количества секунд, минут или часов, вы можете перевести её в число с помощью функции toUInt32, затем округлить число с помощью функции intDiv и умножения, а затем перевести обратно, с помощью функции toDateTime. -### toStartOfHour +## toStartOfHour Округляет дату-с-временем вниз до начала часа. -### toTime +## toTime Переводит дату-с-временем на некоторую фиксированную дату, сохраняя при этом время. -### toRelativeYearNum +## toRelativeYearNum Переводит дату-с-временем или дату в номер года, начиная с некоторого фиксированного момента в прошлом. -### toRelativeMonthNum +## toRelativeMonthNum Переводит дату-с-временем или дату в номер месяца, начиная с некоторого фиксированного момента в прошлом. -### toRelativeWeekNum +## toRelativeWeekNum Переводит дату-с-временем или дату в номер недели, начиная с некоторого фиксированного момента в прошлом. -### toRelativeDayNum +## toRelativeDayNum Переводит дату-с-временем или дату в номер дня, начиная с некоторого фиксированного момента в прошлом. -### toRelativeHourNum +## toRelativeHourNum Переводит дату-с-временем в номер часа, начиная с некоторого фиксированного момента в прошлом. -### toRelativeMinuteNum +## toRelativeMinuteNum Переводит дату-с-временем в номер минуты, начиная с некоторого фиксированного момента в прошлом. -### toRelativeSecondNum +## toRelativeSecondNum Переводит дату-с-временем в номер секунды, начиная с некоторого фиксированного момента в прошлом. -### now +## now Принимает ноль аргументов и возвращает текущее время на один из моментов выполнения запроса. Функция возвращает константу, даже если запрос выполнялся долго. -### today +## today Принимает ноль аргументов и возвращает текущую дату на один из моментов выполнения запроса. То же самое, что toDate(now()) -### yesterday +## yesterday Принимает ноль аргументов и возвращает вчерашнюю дату на один из моментов выполнения запроса. Делает то же самое, что today() - 1. -### timeSlot +## timeSlot Округляет время до получаса. Эта функция является специфичной для Яндекс.Метрики, так как пол часа - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит. -### timeSlots(StartTime, Duration) +## timeSlots(StartTime, Duration) Для интервала времени, начинающегося в StartTime и продолжающегося Duration секунд, возвращает массив моментов времени, состоящий из округлений вниз до получаса точек из этого интервала. Например, `timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. Это нужно для поиска хитов, входящих в соответствующий визит. diff --git a/docs/ru/functions/encoding_functions.md b/docs/ru/functions/encoding_functions.md index b73d0c80f22..f0961822ec3 100644 --- a/docs/ru/functions/encoding_functions.md +++ b/docs/ru/functions/encoding_functions.md @@ -1,22 +1,21 @@ -Функции кодирования -=================== +# Функции кодирования -### hex +## hex Принимает аргументы типов: `String`, `unsigned integer`, `Date`, or `DateTime`. Возвращает строку, содержащую шестнадцатеричное представление аргумента. Используются заглавные буквы `A-F`. Не используются префиксы `0x` и суффиксы `h`. Для строк просто все байты кодируются в виде двух шестнадцатеричных цифр. Числа выводятся в big endian ("человеческом") формате. Для чисел вырезаются старшие нули, но только по целым байтам. Например, `hex(1) = '01'`. `Date` кодируется как число дней с начала unix-эпохи. `DateTime` кодируются как число секунд с начала unix-эпохи. -### unhex(str) +## unhex(str) Принимает строку, содержащую произвольное количество шестнадцатеричных цифр, и возвращает строку, содержащую соответствующие байты. Поддерживаются как строчные, так и заглавные буквы A-F. Число шестнадцатеричных цифр не обязано быть чётным. Если оно нечётное - последняя цифра интерпретируется как младшая половинка байта 00-0F. Если строка-аргумент содержит что-либо кроме шестнадцатеричных цифр, то будет возвращён какой-либо implementation-defined результат (не кидается исключение). Если вы хотите преобразовать результат в число, то вы можете использовать функции reverse и reinterpretAsType. -### UUIDStringToNum(str) +## UUIDStringToNum(str) Принимает строку, содержащую 36 символов в формате `123e4567-e89b-12d3-a456-426655440000`, и возвращает в виде набора байт в FixedString(16). -### UUIDNumToString(str) +## UUIDNumToString(str) Принимает значение типа FixedString(16). Возвращает строку из 36 символов в текстовом виде. -### bitmaskToList(num) +## bitmaskToList(num) Принимает целое число. Возвращает строку, содержащую список степеней двойки, в сумме дающих исходное число; по возрастанию, в текстовом виде, через запятую, без пробелов. -### bitmaskToArray(num) +## bitmaskToArray(num) Принимает целое число. Возвращает массив чисел типа UInt64, содержащий степени двойки, в сумме дающих исходное число; числа в массиве идут по возрастанию. diff --git a/docs/ru/functions/ext_dict_functions.md b/docs/ru/functions/ext_dict_functions.md index 7bec19dd002..946d90aa36f 100644 --- a/docs/ru/functions/ext_dict_functions.md +++ b/docs/ru/functions/ext_dict_functions.md @@ -1,39 +1,38 @@ -Функции для работы с внешними словарями -======================================= +# Функции для работы с внешними словарями Информация о подключении и настройке внешних словарей смотрите в разделе "[Внешние словари](../dicts/external_dicts.md#dicts-external_dicts)". -### dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 +## dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 -### dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 +## dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 -### dictGetFloat32, dictGetFloat64 +## dictGetFloat32, dictGetFloat64 -### dictGetDate, dictGetDateTime +## dictGetDate, dictGetDateTime -### dictGetUUID +## dictGetUUID -### dictGetString +## dictGetString `dictGetT('dict_name', 'attr_name', id)` - получить из словаря dict_name значение атрибута attr_name по ключу id. `dict_name` и `attr_name` - константные строки. `id` должен иметь тип UInt64. Если ключа `id` нет в словаре - вернуть значение по умолчанию, заданное в описании словаря. -### dictGetTOrDefault +## dictGetTOrDefault `dictGetT('dict_name', 'attr_name', id, default)` Аналогично функциям dictGetT, но значение по умолчанию берётся из последнего аргумента функции. -### dictIsIn +## dictIsIn `dictIsIn('dict_name', child_id, ancestor_id)` - для иерархического словаря dict_name - узнать, находится ли ключ child_id внутри ancestor_id (или совпадает с ancestor_id). Возвращает UInt8. -### dictGetHierarchy +## dictGetHierarchy `dictGetHierarchy('dict_name', id)` - для иерархического словаря dict_name - вернуть массив ключей словаря, начиная с id и продолжая цепочкой родительских элементов. Возвращает Array(UInt64). -### dictHas +## dictHas `dictHas('dict_name', id)` - проверить наличие ключа в словаре. Возвращает значение типа UInt8, равное 0, если ключа нет и 1, если ключ есть. diff --git a/docs/ru/functions/hash_functions.md b/docs/ru/functions/hash_functions.md index 2a2cbe554ad..905b14182c4 100644 --- a/docs/ru/functions/hash_functions.md +++ b/docs/ru/functions/hash_functions.md @@ -1,57 +1,56 @@ -Функции хэширования -=================== +# Функции хэширования Функции хэширования могут использоваться для детерминированного псевдослучайного разбрасывания элементов. -### halfMD5 +## halfMD5 Вычисляет MD5 от строки. Затем берёт первые 8 байт от хэша и интерпретирует их как UInt64 в big endian. Принимает аргумент типа String. Возвращает UInt64. Функция работает достаточно медленно (5 миллионов коротких строк в секунду на одном процессорном ядре). Если вам не нужен конкретно MD5, то используйте вместо этого функцию sipHash64. -### MD5 +## MD5 Вычисляет MD5 от строки и возвращает полученный набор байт в виде FixedString(16). Если вам не нужен конкретно MD5, а нужен неплохой криптографический 128-битный хэш, то используйте вместо этого функцию sipHash128. Если вы хотите получить такой же результат, как выдаёт утилита md5sum, напишите lower(hex(MD5(s))). -### sipHash64 +## sipHash64 Вычисляет SipHash от строки. Принимает аргумент типа String. Возвращает UInt64. SipHash - криптографическая хэш-функция. Работает быстрее чем MD5 не менее чем в 3 раза. Подробнее смотрите по ссылке: -### sipHash128 +## sipHash128 Вычисляет SipHash от строки. Принимает аргумент типа String. Возвращает FixedString(16). Отличается от sipHash64 тем, что финальный xor-folding состояния делается только до 128 бит. -### cityHash64 +## cityHash64 Вычисляет CityHash64 от строки или похожую хэш-функцию для произвольного количества аргументов произвольного типа. Если аргумент имеет тип String, то используется CityHash. Это быстрая некриптографическая хэш-функция неплохого качества для строк. Если аргумент имеет другой тип, то используется implementation specific быстрая некриптографическая хэш-функция неплохого качества. Если передано несколько аргументов, то функция вычисляется по тем же правилам, с помощью комбинации по цепочке с использованием комбинатора из CityHash. Например, так вы можете вычислить чексумму всей таблицы с точностью до порядка строк: `SELECT sum(cityHash64(*)) FROM table`. -### intHash32 +## intHash32 Вычисляет 32-битный хэш-код от целого числа любого типа. Это сравнительно быстрая некриптографическая хэш-функция среднего качества для чисел. -### intHash64 +## intHash64 Вычисляет 64-битный хэш-код от целого числа любого типа. Работает быстрее, чем intHash32. Качество среднее. -### SHA1 +## SHA1 -### SHA224 +## SHA224 -### SHA256 +## SHA256 Вычисляет SHA-1, SHA-224, SHA-256 от строки и возвращает полученный набор байт в виде FixedString(20), FixedString(28), FixedString(32). Функция работает достаточно медленно (SHA-1 - примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 - примерно 2.2 миллионов). Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хэш-функция и вы не можете её выбрать. Даже в этих случаях, рекомендуется применять функцию оффлайн - заранее вычисляя значения при вставке в таблицу, вместо того, чтобы применять её при SELECT-ах. -### URLHash(url\[, N\]) +## URLHash(url\[, N\]) Быстрая некриптографическая хэш-функция неплохого качества для строки, полученной из URL путём некоторой нормализации. `URLHash(s)` - вычислить хэш от строки без одного завершающего символа `/`, `?` или `#` на конце, если такой там есть. `URLHash(s, N)` - вычислить хэш от строки до N-го уровня в иерархии URL, без одного завершающего символа `/`, `?` или `#` на конце, если такой там есть. diff --git a/docs/ru/functions/higher_order_functions.md b/docs/ru/functions/higher_order_functions.md index edfa0a5e06d..5e42a726698 100644 --- a/docs/ru/functions/higher_order_functions.md +++ b/docs/ru/functions/higher_order_functions.md @@ -1,8 +1,6 @@ -Функции высшего порядка -======================= +# Функции высшего порядка -Оператор `->`, функция lambda(params, expr) --------------------------------------------- +## Оператор `->`, функция lambda(params, expr) Позволяет описать лямбда-функцию для передачи в функцию высшего порядка. Слева от стрелочки стоит формальный параметр - произвольный идентификатор, или несколько формальных параметров - произвольные идентификаторы в кортеже. Справа от стрелочки стоит выражение, в котором могут использоваться эти формальные параметры, а также любые столбцы таблицы. diff --git a/docs/ru/functions/in_functions.md b/docs/ru/functions/in_functions.md index b04d4d46c0f..71aa8187e4f 100644 --- a/docs/ru/functions/in_functions.md +++ b/docs/ru/functions/in_functions.md @@ -1,15 +1,14 @@ -Функции для реализации оператора IN. -==================================== +# Функции для реализации оператора IN. -### in, notIn, globalIn, globalNotIn +## in, notIn, globalIn, globalNotIn Смотрите раздел "Операторы IN". -### tuple(x, y, ...), оператор (x, y, ...) +## tuple(x, y, ...), оператор (x, y, ...) Функция, позволяющая сгруппировать несколько столбцов. Для столбцов, имеющих типы T1, T2, ... возвращает кортеж типа Tuple(T1, T2, ...), содержащий эти столбцы. Выполнение функции ничего не стоит. Кортежи обычно используются как промежуточное значение в качестве аргумента операторов IN, или для создания списка формальных параметров лямбда-функций. Кортежи не могут быть записаны в таблицу. -### tupleElement(tuple, n), оператор x.N +## tupleElement(tuple, n), оператор x.N Функция, позволяющая достать столбец из кортежа. N - индекс столбца начиная с 1. N должно быть константой. N должно быть целым строго положительным числом не большим размера кортежа. Выполнение функции ничего не стоит. diff --git a/docs/ru/functions/index.md b/docs/ru/functions/index.md index 8f07c442153..9ddcc38f1fc 100644 --- a/docs/ru/functions/index.md +++ b/docs/ru/functions/index.md @@ -1,5 +1,4 @@ -Функции -======= +# Функции Функции бывают как минимум\* двух видов - обычные функции (называются просто, функциями) и агрегатные функции. Это совершенно разные вещи. Обычные функции работают так, как будто применяются к каждой строке по отдельности (для каждой строки, результат вычисления функции не зависит от других строк). Агрегатные функции аккумулируют множество значений из разных строк (то есть, зависят от целого множества строк). @@ -14,23 +13,19 @@ * ``` -Строгая типизация ------------------ +## Строгая типизация В ClickHouse, в отличие от стандартного SQL, типизация является строгой. То есть, не производится неявных преобразований между типами. Все функции работают для определённого набора типов. Это значит, что иногда вам придётся использовать функции преобразования типов. -Склейка одинаковых выражений ----------------------------- +## Склейка одинаковых выражений Все выражения в запросе, имеющие одинаковые AST (одинаковую запись или одинаковый результат синтаксического разбора), считаются имеющими одинаковые значения. Такие выражения склеиваются и исполняются один раз. Одинаковые подзапросы тоже склеиваются. -Типы результата ---------------- +## Типы результата Все функции возвращают одно (не несколько, не ноль) значение в качестве результата. Тип результата обычно определяется только типами аргументов, но не значениями аргументов. Исключение - функция tupleElement (оператор a.N), а также функция toFixedString. -Константы ---------- +## Константы Для простоты, некоторые функции могут работать только с константами в качестве некоторых аргументов. Например, правый аргумент оператора LIKE должен быть константой. Почти все функции возвращают константу для константных аргументов. Исключение - функции генерации случайных чисел. @@ -39,24 +34,20 @@ Функции могут быть по-разному реализованы для константных и не константных аргументов (выполняется разный код). Но результат работы для константы и полноценного столбца, содержащего только одно такое же значение, должен совпадать. -Неизменяемость --------------- +## Неизменяемость Функции не могут поменять значения своих аргументов - любые изменения возвращаются в качестве результата. Соответственно, от порядка записи функций в запросе, результат вычислений отдельных функций не зависит. -Обработка ошибок ----------------- +## Обработка ошибок Некоторые функции могут кидать исключения в случае ошибочных данных. В этом случае, выполнение запроса прерывается, и текст ошибки выводится клиенту. При распределённой обработке запроса, при возникновении исключения на одном из серверов, на другие серверы пытается отправиться просьба тоже прервать выполнение запроса. -Вычисление выражений-аргументов -------------------------------- +## Вычисление выражений-аргументов В почти всех языках программирования, для некоторых операторов может не вычисляться один из аргументов. Обычно - для операторов `&&`, `||`, `?:`. Но в ClickHouse, аргументы функций (операторов) вычисляются всегда. Это связано с тем, что вычисления производятся не по отдельности для каждой строки, а сразу для целых кусочков столбцов. -Выполнение функций при распределённой обработке запроса -------------------------------------------------------- +## Выполнение функций при распределённой обработке запроса При распределённой обработке запроса, как можно большая часть стадий выполнения запроса производится на удалённых серверах, а оставшиеся стадии (слияние промежуточных результатов и всё, что дальше) - на сервере-инициаторе запроса. diff --git a/docs/ru/functions/ip_address_functions.md b/docs/ru/functions/ip_address_functions.md index eb6a4f43e65..5d1945d5545 100644 --- a/docs/ru/functions/ip_address_functions.md +++ b/docs/ru/functions/ip_address_functions.md @@ -1,13 +1,12 @@ -Функции для работы с IP-адресами -================================ +# Функции для работы с IP-адресами -### IPv4NumToString(num) +## IPv4NumToString(num) Принимает число типа UInt32. Интерпретирует его, как IPv4-адрес в big endian. Возвращает строку, содержащую соответствующий IPv4-адрес в формате A.B.C.D (числа в десятичной форме через точки). -### IPv4StringToNum(s) +## IPv4StringToNum(s) Функция, обратная к IPv4NumToString. Если IPv4 адрес в неправильном формате, то возвращает 0. -### IPv4NumToStringClassC(num) +## IPv4NumToStringClassC(num) Похоже на IPv4NumToString, но вместо последнего октета используется xxx. Пример: @@ -105,6 +104,6 @@ LIMIT 10 └────────────────────────────┴────────┘ ``` -### IPv6StringToNum(s) +## IPv6StringToNum(s) Функция, обратная к IPv6NumToString. Если IPv6 адрес в неправильном формате, то возвращает строку из нулевых байт. HEX может быть в любом регистре. diff --git a/docs/ru/functions/json_functions.md b/docs/ru/functions/json_functions.md index 024533142a4..302bed008ef 100644 --- a/docs/ru/functions/json_functions.md +++ b/docs/ru/functions/json_functions.md @@ -1,5 +1,4 @@ -Функции для работы с JSON. -========================== +# Функции для работы с JSON. В Яндекс.Метрике пользователями передаётся JSON в качестве параметров визитов. Для работы с таким JSON-ом, реализованы некоторые функции. (Хотя в большинстве случаев, JSON-ы дополнительно обрабатываются заранее, и полученные значения кладутся в отдельные столбцы в уже обработанном виде.) Все эти функции исходят из сильных допущений о том, каким может быть JSON, и при этом стараются почти ничего не делать. @@ -10,22 +9,22 @@ 3. Поля ищутся на любом уровне вложенности, без разбора. Если есть несколько подходящих полей - берётся первое. 4. В JSON-е нет пробельных символов вне строковых литералов. -### visitParamHas(params, name) +## visitParamHas(params, name) Проверить наличие поля с именем name. -### visitParamExtractUInt(params, name) +## visitParamExtractUInt(params, name) Распарсить UInt64 из значения поля с именем name. Если поле строковое - попытаться распарсить число из начала строки. Если такого поля нет, или если оно есть, но содержит не число, то вернуть 0. -### visitParamExtractInt(params, name) +## visitParamExtractInt(params, name) Аналогично для Int64. -### visitParamExtractFloat(params, name) +## visitParamExtractFloat(params, name) Аналогично для Float64. -### visitParamExtractBool(params, name) +## visitParamExtractBool(params, name) Распарсить значение true/false. Результат - UInt8. -### visitParamExtractRaw(params, name) +## visitParamExtractRaw(params, name) Вернуть значение поля, включая разделители. Примеры: @@ -35,7 +34,7 @@ visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"' visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}' ``` -### visitParamExtractString(params, name) +## visitParamExtractString(params, name) Распарсить строку в двойных кавычках. У значения убирается экранирование. Если убрать экранированные символы не удалось, то возвращается пустая строка. Примеры: diff --git a/docs/ru/functions/logical_functions.md b/docs/ru/functions/logical_functions.md index 5b0676fe0e9..458356b5cc9 100644 --- a/docs/ru/functions/logical_functions.md +++ b/docs/ru/functions/logical_functions.md @@ -1,14 +1,13 @@ -Логические функции -================== +# Логические функции Логические функции принимают любые числовые типы, а возвращают число типа UInt8, равное 0 или 1. Ноль в качестве аргумента считается "ложью", а любое ненулевое значение - "истиной". -### and, оператор AND +## and, оператор AND -### or, оператор OR +## or, оператор OR -### not, оператор NOT +## not, оператор NOT -### xor +## xor diff --git a/docs/ru/functions/math_functions.md b/docs/ru/functions/math_functions.md index 79fb5a9e96c..ad6f6faade1 100644 --- a/docs/ru/functions/math_functions.md +++ b/docs/ru/functions/math_functions.md @@ -1,40 +1,39 @@ -Математические функции -====================== +# Математические функции Все функции возвращают число типа Float64. Точность результата близка к максимально возможной, но результат может не совпадать с наиболее близким к соответствующему вещественному числу машинно представимым числом. -### e() +## e() Принимает ноль аргументов, возвращает число типа Float64, близкое к числу e. -### pi() +## pi() Принимает ноль аргументов, возвращает число типа Float64, близкое к числу π. -### exp(x) +## exp(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к экспоненте от аргумента. -### log(x) +## log(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к натуральному логарифму от аргумента. -### exp2(x) +## exp2(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к 2x. -### log2(x) +## log2(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к двоичному логарифму от аргумента. -### exp10(x) +## exp10(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к 10x. -### log10(x) +## log10(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к десятичному логарифму от аргумента. -### sqrt(x) +## sqrt(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к квадратному корню от аргумента. -### cbrt(x) +## cbrt(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к кубическому корню от аргумента. -### erf(x) +## erf(x) Если x неотрицательно, то erf(x / σ√2) - вероятность того, что случайная величина, имеющая нормальное распределение со среднеквадратичным отклонением σ, принимает значение, отстоящее от мат. ожидания больше чем на x. @@ -50,32 +49,32 @@ SELECT erf(3 / sqrt(2)) └─────────────────────────┘ ``` -### erfc(x) +## erfc(x) Принимает числовой аргумент, возвращает число типа Float64, близкое к 1 - erf(x), но без потери точности для больших x. -### lgamma(x) +## lgamma(x) Логарифм от гамма функции. -### tgamma(x) +## tgamma(x) Гамма функция. -### sin(x) +## sin(x) Синус. -### cos(x) +## cos(x) Косинус. -### tan(x) +## tan(x) Тангенс. -### asin(x) +## asin(x) Арксинус. -### acos(x) +## acos(x) Арккосинус. -### atan(x) +## atan(x) Арктангенс. -### pow(x, y) +## pow(x, y) xy. diff --git a/docs/ru/functions/other_functions.md b/docs/ru/functions/other_functions.md index 8b4070f2a51..6ec205bc5bd 100644 --- a/docs/ru/functions/other_functions.md +++ b/docs/ru/functions/other_functions.md @@ -1,51 +1,50 @@ -Прочие функции -============== +# Прочие функции -### hostName() +## hostName() Возвращает строку - имя хоста, на котором эта функция была выполнена. При распределённой обработке запроса, это будет имя хоста удалённого сервера, если функция выполняется на удалённом сервере. -### visibleWidth(x) +## visibleWidth(x) Вычисляет приблизительную ширину при выводе значения в текстовом (tab-separated) виде на консоль. Функция используется системой для реализации Pretty форматов. -### toTypeName(x) +## toTypeName(x) Возвращает строку, содержащую имя типа переданного аргумента. -### blockSize() +## blockSize() Получить размер блока. В ClickHouse выполнение запроса всегда идёт по блокам (наборам кусочков столбцов). Функция позволяет получить размер блока, для которого её вызвали. -### materialize(x) +## materialize(x) Превращает константу в полноценный столбец, содержащий только одно значение. В ClickHouse полноценные столбцы и константы представлены в памяти по-разному. Функции по-разному работают для аргументов-констант и обычных аргументов (выполняется разный код), хотя результат почти всегда должен быть одинаковым. Эта функция предназначена для отладки такого поведения. -### ignore(...) +## ignore(...) Принимает любые аргументы, всегда возвращает 0. При этом, аргумент всё равно вычисляется. Это может использоваться для бенчмарков. -### sleep(seconds) +## sleep(seconds) Спит seconds секунд на каждый блок данных. Можно указать как целое число, так и число с плавающей запятой. -### currentDatabase() +## currentDatabase() Возвращает имя текущей базы данных. Эта функция может использоваться в параметрах движка таблицы в запросе CREATE TABLE там, где нужно указать базу данных. -### isFinite(x) +## isFinite(x) Принимает Float32 или Float64 и возвращает UInt8, равный 1, если аргумент не бесконечный и не NaN, иначе 0. -### isInfinite(x) +## isInfinite(x) Принимает Float32 или Float64 и возвращает UInt8, равный 1, если аргумент бесконечный, иначе 0. Отметим, что в случае NaN возвращается 0. -### isNaN(x) +## isNaN(x) Принимает Float32 или Float64 и возвращает UInt8, равный 1, если аргумент является NaN, иначе 0. -### hasColumnInTable(\['hostname'\[, 'username'\[, 'password'\]\],\] 'database', 'table', 'column') +## hasColumnInTable(\['hostname'\[, 'username'\[, 'password'\]\],\] 'database', 'table', 'column') Принимает константные строки - имя базы данных, имя таблицы и название столбца. Возвращает константное выражение типа UInt8, равное 1, если есть столбец, иначе 0. Если задан параметр hostname, проверка будет выполнена на удалённом сервере. Функция кидает исключение, если таблица не существует. Для элементов вложенной структуры данных функция проверяет существование столбца. Для самой же вложенной структуры данных функция возвращает 0. -### bar +## bar Позволяет построить unicode-art диаграмму. @@ -98,7 +97,7 @@ ORDER BY h ASC -### transform +## transform Преобразовать значение согласно явно указанному отображению одних элементов на другие. Имеется два варианта функции: @@ -180,7 +179,7 @@ LIMIT 10 └────────────────┴─────────┘ ``` -### formatReadableSize(x) +## formatReadableSize(x) Принимает размер (число байт). Возвращает округленный размер с суффиксом (KiB, MiB и т.д.) в виде строки. Пример: @@ -200,22 +199,22 @@ SELECT └────────────────┴────────────┘ ``` -### least(a, b) +## least(a, b) Возвращает наименьшее значение из a и b. -### greatest(a, b) +## greatest(a, b) Возвращает наибольшее значение из a и b. -### uptime() +## uptime() Возвращает аптайм сервера в секундах. -### version() +## version() Возвращает версию сервера в виде строки. -### rowNumberInAllBlocks() +## rowNumberInAllBlocks() Возвращает порядковый номер строки в блоке данных. Функция учитывает только задействованные блоки данных. -### runningDifference(x) +## runningDifference(x) Считает разницу между последовательными значениями строк в блоке данных. Возвращает 0 для первой строки и разницу с предыдущей строкой для каждой последующей строки. @@ -251,11 +250,11 @@ FROM └─────────┴─────────────────────┴───────┘ ``` -### MACNumToString(num) +## MACNumToString(num) Принимает число типа UInt64. Интерпретирует его, как MAC-адрес в big endian. Возвращает строку, содержащую соответствующий MAC-адрес в формате AA:BB:CC:DD:EE:FF (числа в шестнадцатеричной форме через двоеточие). -### MACStringToNum(s) +## MACStringToNum(s) Функция, обратная к MACNumToString. Если MAC адрес в неправильном формате, то возвращает 0. -### MACStringToOUI(s) +## MACStringToOUI(s) Принимает MAC адрес в формате AA:BB:CC:DD:EE:FF (числа в шестнадцатеричной форме через двоеточие). Возвращает первые три октета как число в формате UInt64. Если MAC адрес в неправильном формате, то возвращает 0. diff --git a/docs/ru/functions/random_functions.md b/docs/ru/functions/random_functions.md index b9edbfc8e1e..5e74ac10b56 100644 --- a/docs/ru/functions/random_functions.md +++ b/docs/ru/functions/random_functions.md @@ -1,5 +1,4 @@ -Функции генерации псевдослучайных чисел -======================================= +# Функции генерации псевдослучайных чисел Используются некриптографические генераторы псевдослучайных чисел. @@ -7,10 +6,10 @@ В случае, если передан аргумент - он может быть любого типа, и его значение никак не используется. Этот аргумент нужен только для того, чтобы предотвратить склейку одинаковых выражений - чтобы две разные записи одной функции возвращали разные столбцы, с разными случайными числами. -### rand +## rand Возвращает псевдослучайное число типа UInt32, равномерно распределённое среди всех чисел типа UInt32. Используется linear congruential generator. -### rand64 +## rand64 Возвращает псевдослучайное число типа UInt64, равномерно распределённое среди всех чисел типа UInt64. Используется linear congruential generator. diff --git a/docs/ru/functions/rounding_functions.md b/docs/ru/functions/rounding_functions.md index 91d29993bb9..a4fc9fa4a05 100644 --- a/docs/ru/functions/rounding_functions.md +++ b/docs/ru/functions/rounding_functions.md @@ -1,7 +1,6 @@ -Функции округления -================== +# Функции округления -### floor(x\[, N\]) +## floor(x\[, N\]) Возвращает наибольшее круглое число, которое меньше или равно, чем x. Круглым называется число, кратное 1 / 10N или ближайшее к нему число соответствующего типа данных, если 1 / 10N не представимо точно. N - целочисленная константа, не обязательный параметр. По умолчанию - ноль, что означает - округлять до целого числа. @@ -13,21 +12,21 @@ N может быть отрицательным. Для целочисленных аргументов имеет смысл округление с отрицательным значением N (для неотрицательных N, функция ничего не делает). В случае переполнения при округлении (например, floor(-128, -1)), возвращается implementation specific результат. -### ceil(x\[, N\]) +## ceil(x\[, N\]) Возвращает наименьшее круглое число, которое больше или равно, чем x. В остальном, аналогично функции floor, см. выше. -### round(x\[, N\]) +## round(x\[, N\]) Возвращает ближайшее к num круглое число, которое может быть меньше или больше или равно x. Если x находится посередине от ближайших круглых чисел, то возвращается какое-либо одно из них (implementation specific). Число -0. может считаться или не считаться круглым (implementation specific). В остальном, аналогично функциям floor и ceil, см. выше. -### roundToExp2(num) +## roundToExp2(num) Принимает число. Если число меньше единицы - возвращает 0. Иначе округляет число вниз до ближайшей (целой неотрицательной) степени двух. -### roundDuration(num) +## roundDuration(num) Принимает число. Если число меньше единицы - возвращает 0. Иначе округляет число вниз до чисел из набора: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. Эта функция специфична для Яндекс.Метрики и предназначена для реализации отчёта по длительности визита. -### roundAge(num) +## roundAge(num) Принимает число. Если число меньше 18 - возвращает 0. Иначе округляет число вниз до чисел из набора: 18, 25, 35, 45, 55. Эта функция специфична для Яндекс.Метрики и предназначена для реализации отчёта по возрасту посетителей. diff --git a/docs/ru/functions/splitting_merging_functions.md b/docs/ru/functions/splitting_merging_functions.md index 25feef16b78..7adf5696838 100644 --- a/docs/ru/functions/splitting_merging_functions.md +++ b/docs/ru/functions/splitting_merging_functions.md @@ -1,19 +1,18 @@ -Функции разбиения и слияния строк и массивов -============================================ +# Функции разбиения и слияния строк и массивов -### splitByChar(separator, s) +## splitByChar(separator, s) Разбивает строку на подстроки, используя в качестве разделителя separator. separator должен быть константной строкой из ровно одного символа. Возвращается массив выделенных подстрок. Могут выделяться пустые подстроки, если разделитель идёт в начале или в конце строки, или если идёт более одного разделителя подряд. -### splitByString(separator, s) +## splitByString(separator, s) То же самое, но использует строку из нескольких символов в качестве разделителя. Строка должна быть непустой. -### arrayStringConcat(arr\[, separator\]) +## arrayStringConcat(arr\[, separator\]) Склеивает строки, перечисленные в массиве, с разделителем separator. separator - необязательный параметр, константная строка, по умолчанию равен пустой строке. Возвращается строка. -### alphaTokens(s) +## alphaTokens(s) Выделяет подстроки из подряд идущих байт из диапазонов a-z и A-Z. Возвращается массив выделенных подстрок. diff --git a/docs/ru/functions/string_functions.md b/docs/ru/functions/string_functions.md index d3adb70348c..43842b59468 100644 --- a/docs/ru/functions/string_functions.md +++ b/docs/ru/functions/string_functions.md @@ -1,61 +1,60 @@ -Функции для работы со строками -============================== +# Функции для работы со строками -### empty +## empty Возвращает 1 для пустой строки, и 0 для непустой строки. Тип результата - UInt8. Строка считается непустой, если содержит хотя бы один байт, пусть даже это пробел или нулевой байт. Функция также работает для массивов. -### notEmpty +## notEmpty Возвращает 0 для пустой строки, и 1 для непустой строки. Тип результата - UInt8. Функция также работает для массивов. -### length +## length Возвращает длину строки в байтах (не символах, не кодовых точках). Тип результата - UInt64. Функция также работает для массивов. -### lengthUTF8 +## lengthUTF8 Возвращает длину строки в кодовых точках Unicode (не символах), при допущении, что строка содержит набор байт, являющийся текстом в кодировке UTF-8. Если допущение не выполнено - то возвращает какой-нибудь результат (не кидает исключение). Тип результата - UInt64. -### lower +## lower Переводит ASCII-символы латиницы в строке в нижний регистр. -### upper +## upper Переводит ASCII-символы латиницы в строке в верхний регистр. -### lowerUTF8 +## lowerUTF8 Переводит строку в нижний регистр, при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Не учитывает язык. То есть, для турецкого языка, результат может быть не совсем верным. Если длина UTF-8 последовательности байт различна для верхнего и нижнего регистра кодовой точки, то для этой кодовой точки, результат работы может быть некорректным. Если строка содержит набор байт, не являющийся UTF-8, то поведение не определено. -### upperUTF8 +## upperUTF8 Переводит строку в верхний регистр, при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Не учитывает язык. То есть, для турецкого языка, результат может быть не совсем верным. Если длина UTF-8 последовательности байт различна для верхнего и нижнего регистра кодовой точки, то для этой кодовой точки, результат работы может быть некорректным. Если строка содержит набор байт, не являющийся UTF-8, то поведение не определено. -### reverse +## reverse Разворачивает строку (как последовательность байт). -### reverseUTF8 +## reverseUTF8 Разворачивает последовательность кодовых точек Unicode, при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Иначе - что-то делает (не кидает исключение). -### concat(s1, s2, ...) +## concat(s1, s2, ...) Склеивает строки, перечисленные в аргументах, без разделителей. -### substring(s, offset, length) +## substring(s, offset, length) Возвращает подстроку, начиная с байта по индексу offset, длины length байт. Индексация символов - начиная с единицы (как в стандартном SQL). Аргументы offset и length должны быть константами. -### substringUTF8(s, offset, length) +## substringUTF8(s, offset, length) Так же, как substring, но для кодовых точек Unicode. Работает при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Если допущение не выполнено - то возвращает какой-нибудь результат (не кидает исключение). -### appendTrailingCharIfAbsent(s, c) +## appendTrailingCharIfAbsent(s, c) Если строка s непустая и не содержит символ c на конце, то добавляет символ c в конец. -### convertCharset(s, from, to) +## convertCharset(s, from, to) Возвращает сконвертированную из кодировки from в кодировку to строку s. diff --git a/docs/ru/functions/string_replace_functions.md b/docs/ru/functions/string_replace_functions.md index ec8ba166c01..7669b598b9f 100644 --- a/docs/ru/functions/string_replace_functions.md +++ b/docs/ru/functions/string_replace_functions.md @@ -1,14 +1,13 @@ -Функции поиска и замены в строках -================================= +# Функции поиска и замены в строках -### replaceOne(haystack, pattern, replacement) +## replaceOne(haystack, pattern, replacement) Замена первого вхождения, если такое есть, подстроки pattern в haystack на подстроку replacement. Здесь и далее, pattern и replacement должны быть константами. -### replaceAll(haystack, pattern, replacement) +## replaceAll(haystack, pattern, replacement) Замена всех вхождений подстроки pattern в haystack на подстроку replacement. -### replaceRegexpOne(haystack, pattern, replacement) +## replaceRegexpOne(haystack, pattern, replacement) Замена по регулярному выражению pattern. Регулярное выражение re2. Заменяется только первое вхождение, если есть. В качестве replacement может быть указан шаблон для замен. Этот шаблон может включать в себя подстановки `\0-\9`. @@ -49,7 +48,7 @@ SELECT replaceRegexpOne('Hello, World!', '.*', '\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0') └────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -### replaceRegexpAll(haystack, pattern, replacement) +## replaceRegexpAll(haystack, pattern, replacement) То же самое, но делается замена всех вхождений. Пример: ```sql diff --git a/docs/ru/functions/string_search_functions.md b/docs/ru/functions/string_search_functions.md index 5bb0bfb3ff0..72f1c9d4d4b 100644 --- a/docs/ru/functions/string_search_functions.md +++ b/docs/ru/functions/string_search_functions.md @@ -1,19 +1,18 @@ -Функции поиска в строках -======================== +# Функции поиска в строках Во всех функциях, поиск регистрозависимый. Во всех функциях, подстрока для поиска или регулярное выражение, должно быть константой. -### position(haystack, needle) +## position(haystack, needle) Поиск подстроки needle в строке haystack. Возвращает позицию (в байтах) найденной подстроки, начиная с 1, или 0, если подстрока не найдена. Есть также функция positionCaseInsensitive. -### positionUTF8(haystack, needle) +## positionUTF8(haystack, needle) Так же, как position, но позиция возвращается в кодовых точках Unicode. Работает при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Если допущение не выполнено - то возвращает какой-нибудь результат (не кидает исключение). Есть также функция positionCaseInsensitiveUTF8. -### match(haystack, pattern) +## match(haystack, pattern) Проверка строки на соответствие регулярному выражению pattern. Регулярное выражение re2. Возвращает 0 (если не соответствует) или 1 (если соответствует). @@ -22,13 +21,13 @@ Регулярное выражение работает со строкой как с набором байт. Регулярное выражение не может содержать нулевые байты. Для шаблонов на поиск подстроки в строке, лучше используйте LIKE или position, так как они работают существенно быстрее. -### extract(haystack, pattern) +## extract(haystack, pattern) Извлечение фрагмента строки по регулярному выражению. Если haystack не соответствует регулярному выражению pattern, то возвращается пустая строка. Если регулярное выражение не содержит subpattern-ов, то вынимается фрагмент, который подпадает под всё регулярное выражение. Иначе вынимается фрагмент, который подпадает под первый subpattern. -### extractAll(haystack, pattern) +## extractAll(haystack, pattern) Извлечение всех фрагментов строки по регулярному выражению. Если haystack не соответствует регулярному выражению pattern, то возвращается пустая строка. Возвращается массив строк, состоящий из всех соответствий регулярному выражению. В остальном, поведение аналогично функции extract (по прежнему, вынимается первый subpattern, или всё выражение, если subpattern-а нет). -### like(haystack, pattern), оператор haystack LIKE pattern +## like(haystack, pattern), оператор haystack LIKE pattern Проверка строки на соответствие простому регулярному выражению. Регулярное выражение может содержать метасимволы `%` и `_`. @@ -41,5 +40,5 @@ Для регулярных выражений вида `%needle%` действует более оптимальный код, который работает также быстро, как функция `position`. Для остальных регулярных выражений, код аналогичен функции match. -### notLike(haystack, pattern), оператор haystack NOT LIKE pattern +## notLike(haystack, pattern), оператор haystack NOT LIKE pattern То же, что like, но с отрицанием. diff --git a/docs/ru/functions/type_conversion_functions.md b/docs/ru/functions/type_conversion_functions.md index f357db31352..7676586db6f 100644 --- a/docs/ru/functions/type_conversion_functions.md +++ b/docs/ru/functions/type_conversion_functions.md @@ -1,19 +1,18 @@ -Функции преобразования типов -============================ +# Функции преобразования типов -### toUInt8, toUInt16, toUInt32, toUInt64 +## toUInt8, toUInt16, toUInt32, toUInt64 -### toInt8, toInt16, toInt32, toInt64 +## toInt8, toInt16, toInt32, toInt64 -### toFloat32, toFloat64 +## toFloat32, toFloat64 -### toUInt8OrZero, toUInt16OrZero, toUInt32OrZero, toUInt64OrZero, toInt8OrZero, toInt16OrZero, toInt32OrZero, toInt64OrZero, toFloat32OrZero, toFloat64OrZero +## toUInt8OrZero, toUInt16OrZero, toUInt32OrZero, toUInt64OrZero, toInt8OrZero, toInt16OrZero, toInt32OrZero, toInt64OrZero, toFloat32OrZero, toFloat64OrZero -### toDate, toDateTime +## toDate, toDateTime -### toString +## toString Функции преобразования между числами, строками (но не фиксированными строками), датами и датами-с-временем. Все эти функции принимают один аргумент. @@ -51,11 +50,11 @@ SELECT Также смотрите функцию `toUnixTimestamp`. -### toFixedString(s, N) +## toFixedString(s, N) Преобразует аргумент типа String в тип FixedString(N) (строку фиксированной длины N). N должно быть константой. Если строка имеет меньше байт, чем N, то она дополняется нулевыми байтами справа. Если строка имеет больше байт, чем N - кидается исключение. -### toStringCutToZero(s) +## toStringCutToZero(s) Принимает аргумент типа String или FixedString. Возвращает String, вырезая содержимое строки до первого найденного нулевого байта. Пример: @@ -80,19 +79,19 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut └────────────┴───────┘ ``` -### reinterpretAsUInt8, reinterpretAsUInt16, reinterpretAsUInt32, reinterpretAsUInt64 +## reinterpretAsUInt8, reinterpretAsUInt16, reinterpretAsUInt32, reinterpretAsUInt64 -### reinterpretAsInt8, reinterpretAsInt16, reinterpretAsInt32, reinterpretAsInt64 +## reinterpretAsInt8, reinterpretAsInt16, reinterpretAsInt32, reinterpretAsInt64 -### reinterpretAsFloat32, reinterpretAsFloat64 +## reinterpretAsFloat32, reinterpretAsFloat64 -### reinterpretAsDate, reinterpretAsDateTime +## reinterpretAsDate, reinterpretAsDateTime Функции принимают строку и интерпретируют байты, расположенные в начале строки, как число в host order (little endian). Если строка имеет недостаточную длину, то функции работают так, как будто строка дополнена необходимым количеством нулевых байт. Если строка длиннее, чем нужно, то лишние байты игнорируются. Дата интерпретируется, как число дней с начала unix-эпохи, а дата-с-временем - как число секунд с начала unix-эпохи. -### reinterpretAsString +## reinterpretAsString Функция принимает число или дату или дату-с-временем и возвращает строку, содержащую байты, представляющие соответствующее значение в host order (little endian). При этом, отбрасываются нулевые байты с конца. Например, значение 255 типа UInt32 будет строкой длины 1 байт. -### CAST(x, t) +## CAST(x, t) Преобразует x в тип данных t. Поддерживается также синтаксис CAST(x AS t). diff --git a/docs/ru/functions/url_functions.md b/docs/ru/functions/url_functions.md index 19e0be6c666..f4697b9537f 100644 --- a/docs/ru/functions/url_functions.md +++ b/docs/ru/functions/url_functions.md @@ -1,61 +1,59 @@ -Функции для работы с URL -======================== +# Функции для работы с URL Все функции работают не по RFC - то есть, максимально упрощены ради производительности. -Функции, извлекающие часть URL-а. ---------------------------------- +## Функции, извлекающие часть URL-а. Если в URL-е нет ничего похожего, то возвращается пустая строка. -#### protocol +### protocol Возвращает протокол. Примеры: http, ftp, mailto, magnet... -#### domain +### domain Возвращает домен. -#### domainWithoutWWW +### domainWithoutWWW Возвращает домен, удалив не более одного 'www.' с начала, если есть. -#### topLevelDomain +### topLevelDomain Возвращает домен верхнего уровня. Пример: .ru. -#### firstSignificantSubdomain +### firstSignificantSubdomain Возвращает "первый существенный поддомен". Это понятие является нестандартным и специфично для Яндекс.Метрики. Первый существенный поддомен - это домен второго уровня, если он не равен одному из com, net, org, co, или домен третьего уровня, иначе. Например, firstSignificantSubdomain('') = 'yandex', firstSignificantSubdomain('') = 'yandex'. Список "несущественных" доменов второго уровня и другие детали реализации могут изменяться в будущем. -#### cutToFirstSignificantSubdomain +### cutToFirstSignificantSubdomain Возвращает часть домена, включающую поддомены верхнего уровня до "первого существенного поддомена" (см. выше). Например, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. -#### path +### path Возвращает путь. Пример: `/top/news.html` Путь не включает в себя query string. -#### pathFull +### pathFull То же самое, но включая query string и fragment. Пример: /top/news.html?page=2\#comments -#### queryString +### queryString Возвращает query-string. Пример: page=1&lr=213. query-string не включает в себя начальный знак вопроса, а также \# и всё, что после \#. -#### fragment +### fragment Возвращает fragment identifier. fragment не включает в себя начальный символ решётки. -#### queryStringAndFragment +### queryStringAndFragment Возвращает query string и fragment identifier. Пример: страница=1\#29390. -#### extractURLParameter(URL, name) +### extractURLParameter(URL, name) Возвращает значение параметра name в URL, если такой есть; или пустую строку, иначе; если параметров с таким именем много - вернуть первый попавшийся. Функция работает при допущении, что имя параметра закодировано в URL в точности таким же образом, что и в переданном аргументе. -#### extractURLParameters(URL) +### extractURLParameters(URL) Возвращает массив строк вида name=value, соответствующих параметрам URL. Значения никак не декодируются. -#### extractURLParameterNames(URL) +### extractURLParameterNames(URL) Возвращает массив строк вида name, соответствующих именам параметров URL. Значения никак не декодируются. -#### URLHierarchy(URL) +### URLHierarchy(URL) Возвращает массив, содержащий URL, обрезанный с конца по символам /, ? в пути и query-string. Подряд идущие символы-разделители считаются за один. Резка производится в позиции после всех подряд идущих символов-разделителей. Пример: -#### URLPathHierarchy(URL) +### URLPathHierarchy(URL) То же самое, но без протокола и хоста в результате. Элемент / (корень) не включается. Пример: Функция используется для реализации древовидных отчётов по URL в Яндекс.Метрике. @@ -67,7 +65,7 @@ URLPathHierarchy('https://example.com/browse/CONV-6788') = ] ``` -#### decodeURLComponent(URL) +### decodeURLComponent(URL) Возвращает декодированный URL. Пример: @@ -81,22 +79,21 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod └────────────────────────────────────────┘ ``` -Функции, удаляющие часть из URL-а ---------------------------------- +## Функции, удаляющие часть из URL-а Если в URL-е нет ничего похожего, то URL остаётся без изменений. -#### cutWWW +### cutWWW Удаляет не более одного 'www.' с начала домена URL-а, если есть. -#### cutQueryString +### cutQueryString Удаляет query string. Знак вопроса тоже удаляется. -#### cutFragment +### cutFragment Удаляет fragment identifier. Символ решётки тоже удаляется. -#### cutQueryStringAndFragment +### cutQueryStringAndFragment Удаляет query string и fragment identifier. Знак вопроса и символ решётки тоже удаляются. -#### cutURLParameter(URL, name) +### cutURLParameter(URL, name) Удаляет параметр URL с именем name, если такой есть. Функция работает при допущении, что имя параметра закодировано в URL в точности таким же образом, что и в переданном аргументе. diff --git a/docs/ru/functions/ym_dict_functions.md b/docs/ru/functions/ym_dict_functions.md index 45ce47c44cc..4e0b6bd451d 100644 --- a/docs/ru/functions/ym_dict_functions.md +++ b/docs/ru/functions/ym_dict_functions.md @@ -1,12 +1,10 @@ -Функции для работы со словарями Яндекс.Метрики -============================================== +# Функции для работы со словарями Яндекс.Метрики Чтобы указанные ниже функции работали, в конфиге сервера должны быть указаны пути и адреса для получения всех словарей Яндекс.Метрики. Словари загружаются при первом вызове любой из этих функций. Если справочники не удаётся загрузить - будет выкинуто исключение. О том, как создать справочники, смотрите в разделе "Словари". -Множественные геобазы ---------------------- +## Множественные геобазы ClickHouse поддерживает работу одновременно с несколькими альтернативными геобазами (иерархиями регионов), для того чтобы можно было поддержать разные точки зрения о принадлежности регионов странам. diff --git a/docs/ru/getting_started/example_datasets/amplab_benchmark.md b/docs/ru/getting_started/example_datasets/amplab_benchmark.md index 3d3a96cffaf..602a12eaea1 100644 --- a/docs/ru/getting_started/example_datasets/amplab_benchmark.md +++ b/docs/ru/getting_started/example_datasets/amplab_benchmark.md @@ -1,5 +1,4 @@ -AMPLab Big Data Benchmark -========================= +# AMPLab Big Data Benchmark См. diff --git a/docs/ru/getting_started/example_datasets/criteo.md b/docs/ru/getting_started/example_datasets/criteo.md index 50952290da6..2d2f44242f8 100644 --- a/docs/ru/getting_started/example_datasets/criteo.md +++ b/docs/ru/getting_started/example_datasets/criteo.md @@ -1,5 +1,4 @@ -Терабайт логов кликов от Criteo -=============================== +# Терабайт логов кликов от Criteo Скачайте данные с diff --git a/docs/ru/getting_started/example_datasets/nyc_taxi.md b/docs/ru/getting_started/example_datasets/nyc_taxi.md index 122db3e71ec..0adbdf050be 100644 --- a/docs/ru/getting_started/example_datasets/nyc_taxi.md +++ b/docs/ru/getting_started/example_datasets/nyc_taxi.md @@ -1,8 +1,6 @@ -Данные о такси в Нью-Йорке -========================== +# Данные о такси в Нью-Йорке -Как импортировать сырые данные ------------------------------- +## Как импортировать сырые данные См. и @@ -30,8 +28,7 @@ mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv ```text time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" - count ------------- +## count 1298979494 (1 row) @@ -277,8 +274,7 @@ WHERE (table = 'trips_mergetree') AND active Между прочим, на MergeTree можно запустить запрос OPTIMIZE. Но это не обязательно, всё будет в порядке и без этого. -Results on single server -======================== +## Results on single server Q1: @@ -365,8 +361,7 @@ Q4: 0.072 sec. В этом случае, время выполнения запросов определяется в первую очередь сетевыми задержками.. Мы выполняли запросы с помощью клиента, расположенного в датацентре Яндекса в Мянтсяля (Финляндия), на кластер в России, что добавляет порядка 20 мс задержки. -Резюме -====== +## Резюме ```text nodes Q1 Q2 Q3 Q4 diff --git a/docs/ru/getting_started/example_datasets/star_schema.md b/docs/ru/getting_started/example_datasets/star_schema.md index 82b8144fdaa..279daed3fdd 100644 --- a/docs/ru/getting_started/example_datasets/star_schema.md +++ b/docs/ru/getting_started/example_datasets/star_schema.md @@ -1,5 +1,4 @@ -Схема «Звезда» -============== +# Схема «Звезда» Компилирование dbgen: diff --git a/docs/ru/getting_started/example_datasets/wikistat.md b/docs/ru/getting_started/example_datasets/wikistat.md index c68b87b7dc7..53f728572a4 100644 --- a/docs/ru/getting_started/example_datasets/wikistat.md +++ b/docs/ru/getting_started/example_datasets/wikistat.md @@ -1,5 +1,4 @@ -WikiStat -======== +# WikiStat См: diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index 834e5661605..454fd0a671a 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -1,8 +1,6 @@ -Начало работы -============= +# Начало работы -Системные требования --------------------- +## Системные требования Система некроссплатформенная. Требуется ОС Linux Ubuntu не более старая, чем Precise (12.04); архитектура x86_64 с поддержкой набора инструкций SSE 4.2. Для проверки наличия SSE 4.2, выполните: @@ -14,8 +12,7 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su Рекомендуется использовать Ubuntu Trusty или Ubuntu Xenial или Ubuntu Precise. Терминал должен работать в кодировке UTF-8 (как по умолчанию в Ubuntu). -Установка ---------- +## Установка В целях тестирования и разработки, система может быть установлена на один сервер или на рабочий компьютер. @@ -79,8 +76,7 @@ RPM пакеты для CentOS, RHEL: -Запуск ------- +## Запуск Для запуска сервера (в качестве демона), выполните: diff --git a/docs/ru/index.md b/docs/ru/index.md index 86a3f33d4e2..0c880e7b14e 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -1,5 +1,4 @@ -Документация ------------- +# Документация ```eval_rst .. toctree:: diff --git a/docs/ru/interfaces/http_interface.md b/docs/ru/interfaces/http_interface.md index f24556965a7..28cb8ce6c66 100644 --- a/docs/ru/interfaces/http_interface.md +++ b/docs/ru/interfaces/http_interface.md @@ -1,5 +1,4 @@ -HTTP-интерфейс -============== +# HTTP-интерфейс HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. @@ -201,8 +200,7 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812 HTTP интерфейс позволяет передать внешние данные (внешние временные таблицы) для использования запроса. Подробнее смотрите раздел "Внешние данные для обработки запроса" -Буферизация ответа ------------------- +## Буферизация ответа Существует возможность включить буферизацию ответа на стороне сервера. Для этого предусмотрены параметры URL `buffer_size` и `wait_end_of_query`. diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md index be4d8e7854d..d2b20ca3767 100644 --- a/docs/ru/interfaces/index.md +++ b/docs/ru/interfaces/index.md @@ -1,7 +1,6 @@ -Интерфейсы -========== +# Интерфейсы Для изучения возможностей системы, загрузки данных в таблицы, ручных запросов, используйте программу clickhouse-client. diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index 037b28ed82b..4cdf4a5769f 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -1,4 +1,3 @@ -JDBC-драйвер -============ +# JDBC-драйвер Для ClickHouse существует официальный JDBC драйвер. Смотрите [здесь](https://github.com/yandex/clickhouse-jdbc) . diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md index ece42550c6b..e73a90ddff3 100644 --- a/docs/ru/interfaces/tcp.md +++ b/docs/ru/interfaces/tcp.md @@ -1,4 +1,3 @@ -Родной интерфейс (TCP) -====================== +# Родной интерфейс (TCP) Родной интерфейс используется в клиенте командной строки clickhouse-client, при межсерверном взаимодействии для распределённой обработки запроса, а также в программах на C++. Будет рассмотрен только клиент командной строки. diff --git a/docs/ru/interfaces/third-party_client_libraries.md b/docs/ru/interfaces/third-party_client_libraries.md index 32a6e4ba162..3a081b9b6ce 100644 --- a/docs/ru/interfaces/third-party_client_libraries.md +++ b/docs/ru/interfaces/third-party_client_libraries.md @@ -1,5 +1,4 @@ -Библиотеки от сторонних разработчиков -===================================== +# Библиотеки от сторонних разработчиков Существуют библиотеки для работы с ClickHouse для: diff --git a/docs/ru/interfaces/third-party_gui.md b/docs/ru/interfaces/third-party_gui.md index 1d37761a6e0..3ee7e7dbc64 100644 --- a/docs/ru/interfaces/third-party_gui.md +++ b/docs/ru/interfaces/third-party_gui.md @@ -1,5 +1,4 @@ -Визуальные интерфейсы от сторонних разработчиков -================================================ +# Визуальные интерфейсы от сторонних разработчиков ## Tabix diff --git a/docs/ru/introduction/features_considered_disadvantages.md b/docs/ru/introduction/features_considered_disadvantages.md index b3b0da5c7a8..c26272f4b6c 100644 --- a/docs/ru/introduction/features_considered_disadvantages.md +++ b/docs/ru/introduction/features_considered_disadvantages.md @@ -1,5 +1,4 @@ -Особенности ClickHouse, которые могут считаться недостатками -============================================================ +# Особенности ClickHouse, которые могут считаться недостатками 1. Отсутствие транзакций. 2. Необходимо, чтобы результат выполнения запроса, в случае агрегации, помещался в оперативку на одном сервере. Объём исходных данных для запроса, при этом, может быть сколь угодно большим. diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index aaa333b0f6a..f513881dcfc 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -1,26 +1,21 @@ -Производительность -================== +# Производительность По результатам внутреннего тестирования, ClickHouse обладает наиболее высокой производительностью (как наиболее высоким throughput на длинных запросах, так и наиболее низкой latency на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на отдельной странице. -Пропускная способность при обработке одного большого запроса ------------------------------------------------------------- +## Пропускная способность при обработке одного большого запроса Пропускную способность можно измерять в строчках в секунду и в мегабайтах в секунду. При условии, что данные помещаются в page cache, не слишком сложный запрос обрабатывается на современном железе со скоростью около 2-10 GB/sec. несжатых данных на одном сервере (в простейшем случае скорость может достигать 30 GB/sec). Если данные не помещаются в page cache, то скорость работы зависит от скорости дисковой подсистемы и коэффициента сжатия данных. Например, если дисковая подсистема позволяет читать данные со скоростью 400 MB/sec., а коэффициент сжатия данных составляет 3, то скорость будет около 1.2GB/sec. Для получения скорости в строчках в секунду, следует поделить скорость в байтах в секунду на суммарный размер используемых в запросе столбцов. Например, если вынимаются столбцы на 10 байт, то скорость будет в районе 100-200 млн. строчек в секунду. При распределённой обработке запроса, скорость обработки запроса растёт почти линейно, но только при условии, что в результате агрегации или при сортировке получается не слишком большое множество строчек. -Задержки при обработке коротких запросов ----------------------------------------- +## Задержки при обработке коротких запросов Если запрос использует первичный ключ, и выбирает для обработки не слишком большое количество строчек (сотни тысяч), и использует не слишком большое количество столбцов, то вы можете рассчитывать на latency менее 50 миллисекунд (от единиц миллисекунд в лучшем случае), при условии, что данные помещаются в page cache. Иначе latency вычисляется из количества seek-ов. Если вы используйте вращающиеся диски, то на не слишком сильно нагруженной системе, latency вычисляется по формуле: seek time (10 мс.) \* количество столбцов в запросе \* количество кусков с данными. -Пропускная способность при обработке большого количества коротких запросов --------------------------------------------------------------------------- +## Пропускная способность при обработке большого количества коротких запросов При тех же условиях, ClickHouse может обработать несколько сотен (до нескольких тысяч в лучшем случае) запросов в секунду на одном сервере. Так как такой сценарий работы не является типичным для аналитических СУБД, рекомендуется рассчитывать не более чем на 100 запросов в секунду. -Производительность при вставке данных -------------------------------------- +## Производительность при вставке данных Данные рекомендуется вставлять пачками не менее 1000 строк или не более одного запроса в секунду. При вставке в таблицу типа MergeTree из tab-separated дампа, скорость вставки будет в районе 50-200 МБ/сек. Если вставляются строчки размером около 1 КБ, то скорость будет в районе 50 000 - 200 000 строчек в секунду. Если строчки маленькие - производительность в строчках в секунду будет выше (на данных БК - `>` 500 000 строк в секунду, на данных Graphite - `>` 1 000 000 строк в секунду). Для увеличения производительности, можно производить несколько запросов INSERT параллельно - при этом производительность растёт линейно. diff --git a/docs/ru/introduction/possible_silly_questions.md b/docs/ru/introduction/possible_silly_questions.md index 50c4fa9f7a7..c01c3be3e45 100644 --- a/docs/ru/introduction/possible_silly_questions.md +++ b/docs/ru/introduction/possible_silly_questions.md @@ -1,8 +1,6 @@ -Возможные глупые вопросы -======================== +# Возможные глупые вопросы -Почему бы не использовать системы типа MapReduce? -------------------------------------------------- +## Почему бы не использовать системы типа MapReduce? Системами типа map-reduce будем называть системы распределённых вычислений, в которых операция reduce сделана на основе распределённой сортировки. Таким образом, к ним относятся YAMR, Hadoop, YT. diff --git a/docs/ru/introduction/what_is_clickhouse.md b/docs/ru/introduction/what_is_clickhouse.md index dc1a0276e0f..1af66427df2 100644 --- a/docs/ru/introduction/what_is_clickhouse.md +++ b/docs/ru/introduction/what_is_clickhouse.md @@ -1,5 +1,4 @@ -Что такое ClickHouse -==================== +# Что такое ClickHouse ClickHouse - столбцовая СУБД для OLAP (Columnar DBMS). diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md index 7b26932703a..24e595b2c49 100644 --- a/docs/ru/introduction/ya_metrika_task.md +++ b/docs/ru/introduction/ya_metrika_task.md @@ -1,13 +1,11 @@ -Постановка задачи в Яндекс.Метрике -================================== +# Постановка задачи в Яндекс.Метрике ClickHouse на данный момент обеспечивает рабту [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Нужно получать произвольные отчёты на основе хитов и визитов, с произвольными сегментами, задаваемыми пользователем. Данные для отчётов обновляются в реальном времени. Запросы должны выполняться сразу (в режиме онлайн). Отчёты должно быть возможно строить за произвольный период. Требуется вычислять сложные агрегаты типа количества уникальных посетителей. На данный момент (апрель 2014), каждый день в Яндекс.Метрику поступает около 12 миллиардов событий (хитов и кликов мыши). Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать сотни миллионов строк за время не более нескольких секунд, или миллионы строк за время не более нескольких сотен миллисекунд. -Использование в Яндекс.Метрике и других отделах Яндекса -------------------------------------------------------- +## Использование в Яндекс.Метрике и других отделах Яндекса В Яндекс.Метрике ClickHouse используется для нескольких задач. Основная задача - построение отчётов в режиме онлайн по неагрегированным данным. Для решения этой задачи используется кластер из 374 серверов, хранящий более 20,3 триллионов строк в базе данных. Объём сжатых данных, без учёта дублирования и репликации, составляет около 2 ПБ. Объём несжатых данных (в формате tsv) составил бы, приблизительно, 17 ПБ. @@ -22,8 +20,7 @@ ClickHouse на данный момент обеспечивает рабту [ ClickHouse имеет более десятка инсталляций в других отделах Яндекса: в Вертикальных сервисах, Маркете, Директе, БК, Бизнес аналитике, Мобильной разработке, AdFox, Персональных сервисах и т п. -Агрегированные и неагрегированные данные ----------------------------------------- +## Агрегированные и неагрегированные данные Существует мнение, что для того, чтобы эффективно считать статистику, данные нужно агрегировать, так как это позволяет уменьшить объём данных. diff --git a/docs/ru/operations/access_rights.md b/docs/ru/operations/access_rights.md index 845c67b0394..7c0166219f5 100644 --- a/docs/ru/operations/access_rights.md +++ b/docs/ru/operations/access_rights.md @@ -1,5 +1,4 @@ -Права доступа -============= +# Права доступа Пользователи и права доступа настраиваются в конфиге пользователей. Обычно это `users.xml`. diff --git a/docs/ru/operations/configuration_files.md b/docs/ru/operations/configuration_files.md index f9c214aaa81..b89afa5bb46 100644 --- a/docs/ru/operations/configuration_files.md +++ b/docs/ru/operations/configuration_files.md @@ -1,7 +1,6 @@ -Конфигурационные файлы -====================== +# Конфигурационные файлы Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 372ef7972b2..afc9fb4f7a5 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,5 +1,4 @@ -Эксплуатация -============ +# Эксплуатация ```eval_rst .. toctree:: diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index 46e3dca06d0..a084017b2ca 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -1,5 +1,4 @@ -Квоты -===== +# Квоты Квоты позволяют ограничить использование ресурсов за некоторый интервал времени, или просто подсчитывать использование ресурсов. Квоты настраиваются в конфиге пользователей. Обычно это users.xml. diff --git a/docs/ru/operations/server_settings/index.md b/docs/ru/operations/server_settings/index.md index be3aff554cb..29edd77efe3 100644 --- a/docs/ru/operations/server_settings/index.md +++ b/docs/ru/operations/server_settings/index.md @@ -1,7 +1,6 @@ -Конфигурационные параметры сервера -================================== +# Конфигурационные параметры сервера Раздел содержит описания настроек сервера, которые не могут изменяться на уровне сессии или запроса. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index 24891797079..513c003a735 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -1,7 +1,8 @@ +# Серверные настройки + -builtin_dictionaries_reload_interval -======================================= +## builtin_dictionaries_reload_interval Интервал (в секундах) перезагрузки встроенных словарей. @@ -17,8 +18,7 @@ ClickHouse перезагружает встроенные словари с з -compression -=========== +## compression Настройки компрессии данных. @@ -63,8 +63,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat -default_database -================= +## default_database База данных по умолчанию. @@ -78,8 +77,7 @@ default_database -default_profile -================ +## default_profile Профиль настроек по умолчанию. @@ -93,8 +91,7 @@ default_profile -dictionaries_config -==================== +## dictionaries_config Путь к конфигурации внешних словарей. @@ -113,8 +110,7 @@ dictionaries_config -dictionaries_lazy_load -======================== +## dictionaries_lazy_load Отложенная загрузка словарей. @@ -132,8 +128,7 @@ dictionaries_lazy_load -format_schema_path -================== +## format_schema_path Путь к каталогу со схемами для входных данных. Например со схемами для формата [CapnProto](../../formats/capnproto.md#format_capnproto). @@ -147,8 +142,7 @@ format_schema_path -graphite -======== +## graphite Отправка даных в [Graphite](https://github.com/graphite-project). @@ -182,8 +176,7 @@ graphite -graphite_rollup -================ +## graphite_rollup Настройка прореживания данных для Graphite. @@ -213,8 +206,7 @@ graphite_rollup -http_port/https_port -====================== +## http_port/https_port Порт для обращений к серверу по протоколу HTTP(s). @@ -230,8 +222,7 @@ http_port/https_port -http_server_default_response -=============================== +## http_server_default_response Страница, показываемая по умолчанию, при обращении к HTTP(s) серверу ClickHouse. @@ -247,8 +238,7 @@ http_server_default_response -include_from -============= +## include_from Путь к файлу с подстановками. @@ -262,8 +252,7 @@ include_from -interserver_http_port -======================= +## interserver_http_port Порт для обмена между серверами ClickHouse. @@ -275,8 +264,7 @@ interserver_http_port -interserver_http_host -======================= +## interserver_http_host Имя хоста, которое могут использовать другие серверы для обращения к этому. @@ -292,8 +280,7 @@ interserver_http_host -keep_alive_timeout -==================== +## keep_alive_timeout Время в миллисекундах, в течение которого ClickHouse ожидает входящих запросов прежде, чем закрыть соединение. @@ -305,8 +292,7 @@ keep_alive_timeout -listen_host -============ +## listen_host Ограничение по хостам, с которых может прийти запрос. Если необходимо, чтобы сервер отвечал всем, то надо указать `::`. @@ -319,8 +305,7 @@ listen_host -logger -====== +## logger Настройки логгирования. @@ -346,8 +331,7 @@ logger -macros -====== +## macros Подстановки параметров реплицируемых таблиц. @@ -363,8 +347,7 @@ macros -mark_cache_size -================= +## mark_cache_size Приблизительный размер (в байтах) кеша "засечек", используемых движками таблиц семейства [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree). @@ -378,8 +361,7 @@ mark_cache_size -max_concurrent_queries -======================== +## max_concurrent_queries Максимальное количество одновременно обрабатываемых запросов. @@ -391,8 +373,7 @@ max_concurrent_queries -max_connections -================ +## max_connections Максимальное количество входящих соединений. @@ -404,8 +385,7 @@ max_connections -max_open_files -================ +## max_open_files Максимальное количество открытых файлов. @@ -421,8 +401,7 @@ max_open_files -max_table_size_to_drop -========================== +## max_table_size_to_drop Ограничение на удаление таблиц. @@ -442,8 +421,7 @@ max_table_size_to_drop -merge_tree -=========== +## merge_tree Тонкая настройка таблиц семейства [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree). @@ -459,8 +437,7 @@ merge_tree -openSSL -======= +## openSSL Настройки клиента/сервера SSL. @@ -521,8 +498,7 @@ openSSL -part_log -========= +## part_log Логгирование событий, связанных с данными типа [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree). Например, события добавления или мержа данных. Лог можно использовать для симуляции алгоритмов слияния, чтобы сравнивать их характеристики. Также, можно визуализировать процесс слияния. @@ -559,8 +535,7 @@ part_log -path -==== +## path Путь к каталогу с данными. @@ -578,8 +553,7 @@ path -query_log -========== +## query_log Настройка логгирования запросов, принятых с настройкой [log_queries=1](#settings-log_queries). @@ -605,8 +579,7 @@ query_log -remote_servers -=============== +## remote_servers Конфигурация кластеров, которые использует движок таблиц Distributed. @@ -622,8 +595,7 @@ remote_servers -resharding -========== +## resharding Путь в ZooKeeper к очереди задач. @@ -639,8 +611,7 @@ resharding -timezone -======== +## timezone Временная зона сервера. @@ -656,8 +627,7 @@ timezone -tcp_port -========= +## tcp_port Порт для взаимодействия с клиентами по протоколу TCP. @@ -669,8 +639,7 @@ tcp_port -tmp_path -========= +## tmp_path Путь ко временным данным для обработки больших запросов. @@ -688,8 +657,7 @@ tmp_path -uncompressed_cache_size -========================= +## uncompressed_cache_size Размер кеша (в байтах) для несжатых данных, используемых движками таблиц семейства [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree). @@ -705,8 +673,7 @@ uncompressed_cache_size -users_config -============= +## users_config Путь к файлу, который содержит: @@ -723,8 +690,7 @@ users_config -zookeeper -========= +## zookeeper Конфигурация серверов ZooKeeper. diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md index c451b6e5d15..1fe9211f922 100644 --- a/docs/ru/operations/settings/index.md +++ b/docs/ru/operations/settings/index.md @@ -1,7 +1,6 @@ -Настройки -========= +# Настройки Все настройки, описанные ниже, могут быть заданы несколькими способами. Настройки задаются послойно, т.е. каждый следующий слой перезаписывает предыдущие настройки. diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md index 0a44a455126..83251f6813c 100644 --- a/docs/ru/operations/settings/query_complexity.md +++ b/docs/ru/operations/settings/query_complexity.md @@ -1,5 +1,4 @@ -Ограничения на сложность запроса -================================ +# Ограничения на сложность запроса Ограничения на сложность запроса - часть настроек. Используются, чтобы обеспечить более безопасное исполнение запросов из пользовательского интерфейса. @@ -18,8 +17,7 @@ -readonly --------- +## readonly При значении 0 можно выполнять любые запросы. При значении 1 можно выполнять только запросы на чтение (например, SELECT и SHOW). Запросы на запись и изменение настроек (INSERT, SET) запрещены. @@ -29,8 +27,7 @@ readonly При использовании метода GET HTTP интерфейса, автоматически выставляется readonly = 1. То есть, для запросов, модифицирующие данные, можно использовать только метод POST. Сам запрос при этом можно отправлять как в теле POST-а, так и в параметре URL. -max_memory_usage ------------------- +## max_memory_usage Максимальное количество потребляемой памяти при выполнении запроса на одном сервере. По умолчанию - 10 GB. @@ -46,166 +43,135 @@ max_memory_usage Потребление оперативки не полностью учитывается для состояний агрегатных функций min, max, any, anyLast, argMin, argMax от аргументов String и Array. -max_rows_to_read -------------------- +## max_rows_to_read Следующие ограничения могут проверяться на каждый блок (а не на каждую строку). То есть, ограничения могут быть немного нарушены. При выполнении запроса в несколько потоков, следующие ограничения действуют в каждом потоке по-отдельности. Максимальное количество строчек, которое можно прочитать из таблицы при выполнении запроса. -max_bytes_to_read --------------------- +## max_bytes_to_read Максимальное количество байт (несжатых данных), которое можно прочитать из таблицы при выполнении запроса. -read_overflow_mode --------------------- +## read_overflow_mode Что делать, когда количество прочитанных данных превысило одно из ограничений: throw или break. По умолчанию: throw. -max_rows_to_group_by ------------------------- +## max_rows_to_group_by Максимальное количество уникальных ключей, получаемых в процессе агрегации. Позволяет ограничить потребление оперативки при агрегации. -group_by_overflow_mode -------------------------- +## group_by_overflow_mode Что делать, когда количество уникальных ключей при агрегации превысило ограничение: throw, break или any. По умолчанию: throw. Использование значения any позволяет выполнить GROUP BY приближённо. Качество такого приближённого вычисления сильно зависит от статистических свойств данных. -max_rows_to_sort -------------------- +## max_rows_to_sort Максимальное количество строк до сортировки. Позволяет ограничить потребление оперативки при сортировке. -max_bytes_to_sort --------------------- +## max_bytes_to_sort Максимальное количество байт до сортировки. -sort_overflow_mode --------------------- +## sort_overflow_mode Что делать, если количество строк, полученное перед сортировкой, превысило одно из ограничений: throw или break. По умолчанию: throw. -max_result_rows ------------------ +## max_result_rows Ограничение на количество строк результата. Проверяются также для подзапросов и на удалённых серверах при выполнении части распределённого запроса. -max_result_bytes ------------------- +## max_result_bytes Ограничение на количество байт результата. Аналогично. -result_overflow_mode ----------------------- +## result_overflow_mode Что делать, если объём результата превысил одно из ограничений: throw или break. По умолчанию: throw. Использование break по смыслу похоже на LIMIT. -max_execution_time --------------------- +## max_execution_time Максимальное время выполнения запроса в секундах. На данный момент не проверяется при одной из стадий сортировки а также при слиянии и финализации агрегатных функций. -timeout_overflow_mode ------------------------ +## timeout_overflow_mode Что делать, если запрос выполняется дольше max_execution_time: throw или break. По умолчанию: throw. -min_execution_speed ---------------------- +## min_execution_speed Минимальная скорость выполнения запроса в строчках в секунду. Проверяется на каждый блок данных по истечении timeout_before_checking_execution_speed. Если скорость выполнения запроса оказывается меньше, то кидается исключение. -timeout_before_checking_execution_speed -------------------------------------------- +## timeout_before_checking_execution_speed Проверять, что скорость выполнения запроса не слишком низкая (не меньше min_execution_speed), после прошествия указанного времени в секундах. -max_columns_to_read ----------------------- +## max_columns_to_read Максимальное количество столбцов, которых можно читать из таблицы в одном запросе. Если запрос требует чтения большего количества столбцов - кинуть исключение. -max_temporary_columns ------------------------ +## max_temporary_columns Максимальное количество временных столбцов, которых необходимо одновременно держать в оперативке, в процессе выполнения запроса, включая константные столбцы. Если временных столбцов оказалось больше - кидается исключение. -max_temporary_non_const_columns ------------------------------------ +## max_temporary_non_const_columns То же самое, что и max_temporary_columns, но без учёта столбцов-констант. Стоит заметить, что столбцы-константы довольно часто образуются в процессе выполнения запроса, но расходуют примерно нулевое количество вычислительных ресурсов. -max_subquery_depth --------------------- +## max_subquery_depth Максимальная вложенность подзапросов. Если подзапросы более глубокие - кидается исключение. По умолчанию: 100. -max_pipeline_depth --------------------- +## max_pipeline_depth Максимальная глубина конвейера выполнения запроса. Соответствует количеству преобразований, которое проходит каждый блок данных в процессе выполнения запроса. Считается в пределах одного сервера. Если глубина конвейера больше - кидается исключение. По умолчанию: 1000. -max_ast_depth ---------------- +## max_ast_depth Максимальная вложенность синтаксического дерева запроса. Если превышена - кидается исключение. На данный момент, проверяются не во время парсинга а уже после парсинга запроса. То есть, во время парсинга может быть создано слишком глубокое синтаксическое дерево, но запрос не будет выполнен. По умолчанию: 1000. -max_ast_elements ------------------- +## max_ast_elements Максимальное количество элементов синтаксического дерева запроса. Если превышено - кидается исключение. Аналогично, проверяется уже после парсинга запроса. По умолчанию: 10 000. -max_rows_in_set ------------------- +## max_rows_in_set Максимальное количество строчек для множества в секции IN, создаваемого из подзапроса. -max_bytes_in_set -------------------- +## max_bytes_in_set Максимальное количество байт (несжатых данных), занимаемое множеством в секции IN, создаваемым из подзапроса. -set_overflow_mode -------------------- +## set_overflow_mode Что делать, когда количество данных превысило одно из ограничений: throw или break. По умолчанию: throw. -max_rows_in_distinct ------------------------ +## max_rows_in_distinct Максимальное количество различных строчек при использовании DISTINCT. -max_bytes_in_distinct ------------------------- +## max_bytes_in_distinct Максимальное количество байт, занимаемых хэш-таблицей, при использовании DISTINCT. -distinct_overflow_mode ------------------------- +## distinct_overflow_mode Что делать, когда количество данных превысило одно из ограничений: throw или break. По умолчанию: throw. -max_rows_to_transfer ------------------------ +## max_rows_to_transfer Максимальное количество строчек, которых можно передать на удалённый сервер или сохранить во временную таблицу, при использовании GLOBAL IN. -max_bytes_to_transfer ------------------------- +## max_bytes_to_transfer Максимальное количество байт (несжатых данных), которых можно передать на удалённый сервер или сохранить во временную таблицу, при использовании GLOBAL IN. -transfer_overflow_mode ------------------------- +## transfer_overflow_mode Что делать, когда количество данных превысило одно из ограничений: throw или break. По умолчанию: throw. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 5ac49b54a43..72b896d1505 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1,7 +1,8 @@ +# Настройки + -distributed_product_mode -========================== +## distributed_product_mode Изменяет поведение [распределенных подзапросов](../../query_language/queries.md#queries-distributed-subrequests), т.е. в тех случаях, когда запрос содержит произведение распределённых таблиц. @@ -17,8 +18,7 @@ ClickHouse применяет настройку в том случае, ког -fallback_to_stale_replicas_for_distributed_queries -======================================================== +## fallback_to_stale_replicas_for_distributed_queries Форсирует запрос в устаревшую реплику в случае, если актуальные данные недоступны. Смотрите "[Репликация](../../table_engines/replication.md#table_engines-replication)". @@ -30,8 +30,7 @@ fallback_to_stale_replicas_for_distributed_queries -force_index_by_date -====================== +## force_index_by_date Запрещает выполнение запросов, если использовать индекс по дате невозможно. @@ -41,8 +40,7 @@ force_index_by_date -force_primary_key -=================== +## force_primary_key Запрещает выполнение запросов, если использовать индекс по первичному ключу невозможно. @@ -52,15 +50,13 @@ force_primary_key -fsync_metadata -=============== +## fsync_metadata Включить или отключить fsync при записи .sql файлов. По-умолчанию включено. Имеет смысл выключать, если на сервере миллионы мелких таблиц-чанков, которые постоянно создаются и уничтожаются. -input_format_allow_errors_num -================================= +## input_format_allow_errors_num Устанавливает максимальное количество допустимых ошибок при чтении из текстовых форматов (CSV, TSV и т.п.). @@ -72,8 +68,7 @@ input_format_allow_errors_num В случае превышения `input_format_allow_errors_num` ClickHouse генерирует исключение. -input_format_allow_errors_ratio -=================================== +## input_format_allow_errors_ratio Устанавливает максимальную долю допустимых ошибок при чтении из текстовых форматов (CSV, TSV и т.п.). Доля ошибок задаётся в виде числа с плавающей запятой от 0 до 1. @@ -86,8 +81,7 @@ input_format_allow_errors_ratio В случае превышения `input_format_allow_errors_ratio` ClickHouse генерирует исключение. -max_block_size -================ +## max_block_size Данные в ClickHouse обрабатываются по блокам (наборам кусочков столбцов). Внутренние циклы обработки одного блока достаточно эффективны, но при этом существуют заметные издержки на каждый блок. `max_block_size` - это рекомендация, какого размера блоки (в количестве строк) загружать из таблицы. Размер блока должен быть не слишком маленьким, чтобы издержки на каждый блок оставались незаметными, и не слишком большим, чтобы запрос с LIMIT-ом, который завершается уже после первого блока, выполнялся быстро; чтобы не использовалось слишком много оперативки при вынимании большого количества столбцов в несколько потоков; чтобы оставалась хоть какая-нибудь кэш-локальность. @@ -95,8 +89,7 @@ max_block_size Из таблицы не всегда загружаются блоки размера `max_block_size`. Если ясно, что нужно прочитать меньше данных, то будет считан блок меньшего размера. -preferred_block_size_bytes -============================= +## preferred_block_size_bytes Служит для тех же целей что и `max_block_size`, но задает реккомедуемый размер блоков в байтах, выбирая адаптивное количество строк в блоке. При этом размер блока не может быть более `max_block_size` строк. @@ -104,8 +97,7 @@ preferred_block_size_bytes -log_queries -============ +## log_queries Установка логгирования запроса. @@ -117,8 +109,7 @@ log_queries -max_insert_block_size -======================== +## max_insert_block_size Формировать блоки указанного размера, при вставке в таблицу. Эта настройка действует только в тех случаях, когда сервер сам формирует такие блоки. @@ -132,8 +123,7 @@ max_insert_block_size -max_replica_delay_for_distributed_queries -============================================== +## max_replica_delay_for_distributed_queries Отключает отстающие реплики при распределенных запросах. Смотрите "[Репликация](../../table_engines/replication.md#table_engines-replication)". @@ -143,8 +133,7 @@ max_replica_delay_for_distributed_queries Используется при выполнении `SELECT` из распределенной таблицы, которая указывает на реплицированные таблицы. -max_threads -============ +## max_threads Максимальное количество потоков обработки запроса - без учёта потоков для чтения данных с удалённых серверов (смотрите параметр max_distributed_connections). @@ -160,15 +149,13 @@ max_threads Чем меньше `max_threads`, тем меньше будет использоваться оперативки. -max_compress_block_size -========================== +## max_compress_block_size Максимальный размер блоков не сжатых данных перед сжатием при записи в таблицу. По умолчанию - 1 048 576 (1 MiB). При уменьшении размера, незначительно уменьшается коэффициент сжатия, незначительно возрастает скорость сжатия и разжатия за счёт кэш-локальности, и уменьшается потребление оперативки. Как правило, не имеет смысла менять эту настройку. Не путайте блоки для сжатия (кусок памяти, состоящий из байт) и блоки для обработки запроса (пачка строк из таблицы). -min_compress_block_size -========================== +## min_compress_block_size Для таблиц типа "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". В целях уменьшения задержек при обработке запросов, блок сжимается при записи следующей засечки, если его размер не меньше min_compress_block_size. По умолчанию - 65 536. @@ -182,43 +169,36 @@ min_compress_block_size Как правило, не имеет смысла менять эту настройку. -max_query_size -================ +## max_query_size Максимальный кусок запроса, который будет считан в оперативку для разбора парсером языка SQL. Запрос INSERT также содержит данные для INSERT-а, которые обрабатываются отдельным, потоковым парсером (расходующим O(1) оперативки), и не учитываются в этом ограничении. По умолчанию - 256 KiB. -interactive_delay -================== +## interactive_delay Интервал в микросекундах для проверки, не запрошена ли остановка выполнения запроса, и отправки прогресса. По умолчанию - 100 000 (проверять остановку запроса и отправлять прогресс десять раз в секунду). -connect_timeout -================ +## connect_timeout -receive_timeout -================ +## receive_timeout -send_timeout -============= +## send_timeout Таймауты в секундах на сокет, по которому идёт общение с клиентом. По умолчанию - 10, 300, 300. -poll_interval -============== +## poll_interval Блокироваться в цикле ожидания запроса в сервере на указанное количество секунд. По умолчанию - 10. -max_distributed_connections -============================= +## max_distributed_connections Максимальное количество одновременных соединений с удалёнными серверами при распределённой обработке одного запроса к одной таблице типа Distributed. Рекомендуется выставлять не меньше, чем количество серверов в кластере. @@ -226,46 +206,40 @@ max_distributed_connections Следующие параметры имеют значение только на момент создания таблицы типа Distributed (и при запуске сервера), поэтому их не имеет смысла менять в рантайме. -distributed_connections_pool_size -==================================== +## distributed_connections_pool_size Максимальное количество одновременных соединений с удалёнными серверами при распределённой обработке всех запросов к одной таблице типа Distributed. Рекомендуется выставлять не меньше, чем количество серверов в кластере. По умолчанию - 128. -connect_timeout_with_failover_ms -==================================== +## connect_timeout_with_failover_ms Таймаут в миллисекундах на соединение с удалённым сервером, для движка таблиц Distributed, если используются секции shard и replica в описании кластера. В случае неуспеха, делается несколько попыток соединений с разными репликами. По умолчанию - 50. -connections_with_failover_max_tries -======================================= +## connections_with_failover_max_tries Максимальное количество попыток соединения с каждой репликой, для движка таблиц Distributed. По умолчанию - 3. -extremes -======== +## extremes Считать ли экстремальные значения (минимумы и максимумы по столбцам результата запроса). Принимает 0 или 1. По умолчанию - 0 (выключено). Подробнее смотрите раздел "Экстремальные значения". -use_uncompressed_cache -======================== +## use_uncompressed_cache Использовать ли кэш разжатых блоков. Принимает 0 или 1. По умолчанию - 0 (выключено). Кэш разжатых блоков (только для таблиц семейства MergeTree) позволяет существенно уменьшить задержки и увеличить пропускную способность при обработке большого количества коротких запросов. Включите эту настройку для пользователей, от которых идут частые короткие запросы. Также обратите внимание на конфигурационный параметр uncompressed_cache_size (настраивается только в конфигурационном файле) - размер кэша разжатых блоков. По умолчанию - 8 GiB. Кэш разжатых блоков заполняется по мере надобности; наиболее невостребованные данные автоматически удаляются. Для запросов, читающих хоть немного приличный объём данных (миллион строк и больше), кэш разжатых блоков автоматически выключается, чтобы оставить место для действительно мелких запросов. Поэтому, можно держать настройку use_uncompressed_cache всегда выставленной в 1. -replace_running_query -======================= +## replace_running_query При использовании HTTP-интерфейса, может быть передан параметр query_id - произвольная строка, являющаяся идентификатором запроса. Если в этот момент, уже существует запрос от того же пользователя с тем же query_id, то поведение определяется параметром replace_running_query. @@ -276,13 +250,13 @@ replace_running_query Эта настройка, выставленная в 1, используется в Яндекс.Метрике для реализации suggest-а значений для условий сегментации. После ввода очередного символа, если старый запрос ещё не выполнился, его следует отменить. -# schema +## schema Параметр применяется в том случае, когда используются форматы, требующие определения схемы, например [Cap'n Proto](https://capnproto.org/). Значение параметра зависит от формата. -# stream_flush_interval_ms +## stream_flush_interval_ms Работает для таблиц со стриммингом в случае тайм-аута, или когда поток генерирует [max_insert_block_size](#settings-settings-max_insert_block_size) строк. @@ -293,8 +267,7 @@ replace_running_query -load_balancing -=============== +## load_balancing На какие реплики (среди живых реплик) предпочитать отправлять запрос (при первой попытке) при распределённой обработке запроса. @@ -315,43 +288,37 @@ load_balancing Реплики перебираются в таком порядке, в каком они указаны. Количество ошибок не имеет значения. Этот способ подходит для тех случаев, когда вы точно знаете, какая реплика предпочтительнее. -totals_mode -============ +## totals_mode Каким образом вычислять TOTALS при наличии HAVING, а также при наличии max_rows_to_group_by и group_by_overflow_mode = 'any'. Смотрите раздел "Модификатор WITH TOTALS". -totals_auto_threshold -======================= +## totals_auto_threshold Порог для `totals_mode = 'auto'`. Смотрите раздел "Модификатор WITH TOTALS". -default_sample -=============== +## default_sample Число с плавающей запятой от 0 до 1. По умолчанию - 1. Позволяет выставить коэффициент сэмплирования по умолчанию для всех запросов SELECT. (Для таблиц, не поддерживающих сэмплирование, будет кидаться исключение.) Если равно 1 - сэмплирование по умолчанию не делается. -max_parallel_replicas -======================= +## max_parallel_replicas Максимальное количество используемых реплик каждого шарда при выполнении запроса. Для консистентности (чтобы получить разные части одного и того же разбиения), эта опция работает только при заданном ключе сэмплирования. Отставание реплик не контролируется. -compile -======= +## compile Включить компиляцию запросов. По умолчанию - 0 (выключено). Компиляция предусмотрена только для части конвейера обработки запроса - для первой стадии агрегации (GROUP BY). В случае, если эта часть конвейера была скомпилирована, запрос может работать быстрее, за счёт разворачивания коротких циклов и инлайнинга вызовов агрегатных функций. Максимальный прирост производительности (до четырёх раз в редких случаях) достигается на запросах с несколькими простыми агрегатными функциями. Как правило, прирост производительности незначителен. В очень редких случаях возможно замедление выполнения запроса. -min_count_to_compile -======================= +## min_count_to_compile После скольких раз, когда скомпилированный кусок кода мог пригодиться, выполнить его компиляцию. По умолчанию - 3. В случае, если значение равно нулю, то компиляция выполняется синхронно, и запрос будет ждать окончания процесса компиляции перед продолжением выполнения. Это можно использовать для тестирования, иначе используйте значения, начиная с 1. Как правило, компиляция занимает по времени около 5-10 секунд. @@ -360,21 +327,18 @@ min_count_to_compile Скомпилированный код требуется для каждого разного сочетания используемых в запросе агрегатных функций и вида ключей в GROUP BY. Результаты компиляции сохраняются в директории build в виде .so файлов. Количество результатов компиляции не ограничено, так как они не занимают много места. При перезапуске сервера, старые результаты будут использованы, за исключением случая обновления сервера - тогда старые результаты удаляются. -input_format_skip_unknown_fields -==================================== +## input_format_skip_unknown_fields Если значение истинно, то при выполнении INSERT из входных данных пропускаются (не рассматриваются) колонки с неизвестными именами, иначе в данной ситуации будет сгенерировано исключение. Работает для форматов JSONEachRow и TSKV. -output_format_json_quote_64bit_integers -============================================ +## output_format_json_quote_64bit_integers Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек. -strict_insert_defaults -======================== +## strict_insert_defaults Строгое присвоение значений по умолчанию при добавлении данных. diff --git a/docs/ru/operations/settings/settings_profiles.md b/docs/ru/operations/settings/settings_profiles.md index eb573a0375a..13a20ea04dc 100644 --- a/docs/ru/operations/settings/settings_profiles.md +++ b/docs/ru/operations/settings/settings_profiles.md @@ -1,5 +1,4 @@ -Профили настроек -================ +# Профили настроек Профили настроек - это множество настроек, сгруппированных под одним именем. Для каждого пользователя ClickHouse указывается некоторый профиль. Все настройки профиля можно применить, установив настройку с именем profile. Пример: diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index 73812cb2b59..81f2aeb87e9 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -1,27 +1,22 @@ -Советы по эксплуатации -====================== +# Советы по эксплуатации -Процессор ---------- +## Процессор Требуется поддержка набора инструкций SSE 4.2. Современные процессоры (с 2008 года) его поддерживают. При выборе между процессорами с большим числом ядер с немного меньшей тактовой частотой и процессором с меньшим числом ядер с высокой тактовой частотой, первый вариант более предпочтителен. Например, 16 ядер с 2600 MHz лучше, чем 8 ядер 3600 MHz. -Hyper-Threading ---------------- +## Hyper-Threading Hyper-threading лучше не отключать. Некоторые запросам он помогает, а некоторым — нет. -Turbo-Boost ------------ +## Turbo-Boost Turbo-Boost крайне не рекомендуется отключать. При типичной нагрузке он значительно улучшает производительность. Можно использовать `turbostat` для просмотра реальной тактовой частоты процессора под нагрузкой. -CPU scaling governor --------------------- +## CPU scaling governor Нужно всегда использовать `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе. @@ -29,26 +24,22 @@ CPU scaling governor sudo echo 'performance' | tee /sys/devices/system/cpu/cpu\*/cpufreq/scaling_governor ``` -Ограничение CPU ---------------- +## Ограничение CPU Процессоры могут перегреваться. С помощью `dmesg` можно увидеть, если тактовая частота процессора была ограничена из-за перегрева. Также ограничение может устанавливаться снаружи на уровне датацентра. С помощью `turbostat` можно за этим наблюдать под нагрузкой. -Оперативная память ------------------- +## Оперативная память Для небольших объемов данных (до \~200 Гб в сжатом виде) лучше всего использовать столько памяти не меньше, чем объем данных. Для больших объемов данных, при выполнении интерактивных (онлайн) запросов, стоит использовать разумный объем оперативной памяти (128 Гб или более) для того, чтобы горячее подмножество данных поместилось в кеше страниц. Даже для объемов данных в \~50 Тб на сервер, использование 128 Гб оперативной памяти намного лучше для производительности выполнения запросов, чем 64 Гб. -Файл подкачки -------------- +## Файл подкачки Всегда отключайте файл подкачки. Единственной причиной этого не делать может быть только использование ClickHouse на личном ноутбуке. -Huge pages ----------- +## Huge pages Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности. @@ -59,8 +50,7 @@ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled С помощью `perf top` можно наблюдать за временем, проведенном в ядре операционной системы для управления памятью. Постоянные huge pages так же не нужно аллоцировать. -Подсистема хранения -------------------- +## Подсистема хранения Если ваш бюджет позволяет использовать SSD, используйте SSD. В противном случае используйте HDD. SATA HDDs 7200 RPM подойдут. @@ -68,8 +58,7 @@ echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled Предпочитайте много серверов с локальными жесткими дисками вместо меньшего числа серверов с подключенными дисковыми полками. Но для хранения архивов с редкими запросами полки всё же подходят. -RAID ----- +## RAID При использовании HDD можно объединить их RAID-10, RAID-5, RAID-6 или RAID-50. Лучше использовать программный RAID в Linux (`mdadm`). Лучше не использовать LVM. @@ -94,29 +83,25 @@ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size Включите NCQ с длинной очередью. Для HDD стоит выбрать планировщик CFQ, а для SSD — noop. Не стоит уменьшать настройку readahead. На HDD стоит включать кеш записи. -Файловая система ----------------- +## Файловая система Ext4 — самый проверенный вариант, стоит указывать опции монтирования `noatime,nobarrier`. XFS также подходит, но не так тщательно протестирована в сочетании с ClickHouse. Большинство других файловых систем также должны нормально работать. Файловые системы с отложенной аллокацией работают лучше. -Ядро Linux ----------- +## Ядро Linux Не используйте слишком старое ядро Linux. В 2015 году 3.18.19 — достаточно свежее. Рассмотрите возможность использования сборки ядра от Яндекса: — это дает прирост в производительности не менее 5%. -Сеть ----- +## Сеть При использовании IPv6, стоит увеличить размер кеша маршрутов. Ядра Linux до 3.2 имели массу проблем в реализации IPv6. Предпочитайте как минимум 10 Гбит сеть. 1 Гбит также будет работать, но намного хуже для починки реплик с десятками терабайт данных или для обработки распределенных запросов с большим объемом промежуточных данных. -ZooKeeper ---------- +## ZooKeeper Вероятно вы уже используете ZooKeeper для других целей. Можно использовать ту же инсталляцию ZooKeeper, если она не сильно перегружена. diff --git a/docs/ru/operators/index.md b/docs/ru/operators/index.md index 687d2afcef9..61d40fb350d 100644 --- a/docs/ru/operators/index.md +++ b/docs/ru/operators/index.md @@ -1,23 +1,19 @@ -Операторы -========= +# Операторы Все операторы преобразуются в соответствующие функции на этапе парсинга запроса, с учётом их приоритетов и ассоциативности. Далее будут перечислены группы операторов в порядке их приоритета (чем выше, тем раньше оператор связывается со своими аргументами). -Операторы доступа ------------------ +## Операторы доступа `a[N]` - доступ к элементу массива, функция `arrayElement(a, N)`. `a.N` - доступ к элементу кортежа, функция `tupleElement(a, N)`. -Оператор числового отрицания ----------------------------- +## Оператор числового отрицания `-a` - функция `negate(a)`. -Операторы умножения и деления ------------------------------ +## Операторы умножения и деления `a * b` - функция `multiply(a, b)` @@ -25,15 +21,13 @@ `a % b` - функция `modulo(a, b)` -Операторы сложения и вычитания ------------------------------- +## Операторы сложения и вычитания `a + b` - функция `plus(a, b)` `a - b` - функция `minus(a, b)` -Операторы сравнения -------------------- +## Операторы сравнения `a = b` - функция `equals(a, b)` @@ -57,8 +51,7 @@ `a BETWEEN b AND c` - равнозначно `a >= b AND a <= c` -Операторы для работы с множествами ----------------------------------- +## Операторы для работы с множествами *Смотрите раздел "Операторы IN".* @@ -70,23 +63,19 @@ `a GLOBAL NOT IN ...` - функция `globalNotIn(a, b)` -Оператор логического отрицания ------------------------------- +## Оператор логического отрицания `NOT a` - функция `not(a)` -Оператор логического "И". -------------------------- +## Оператор логического "И". `a AND b` - функция `and(a, b)` -Оператор логического "ИЛИ". ---------------------------- +## Оператор логического "ИЛИ". `a OR b` - функция `or(a, b)` -Условный оператор ------------------ +## Условный оператор `a ? b : c` - функция `if(a, b, c)` @@ -94,8 +83,7 @@ Условный оператор сначала вычисляет значения b и c, затем проверяет выполнение условия a, и только после этого возвращает соответствующее значение. Если в качестве b или с выступает функция arrayJoin(), то размножение каждой строки произойдет вне зависимости от условия а. -Условное выражение ------------------- +## Условное выражение ```sql CASE [x] @@ -107,30 +95,25 @@ END В случае указания x - функция transform(x, \[a, ...\], \[b, ...\], c). Иначе - multiIf(a, b, ..., c). -Оператор склеивания строк -------------------------- +## Оператор склеивания строк `s1 || s2` - функция `concat(s1, s2)` -Оператор создания лямбда-выражения ----------------------------------- +## Оператор создания лямбда-выражения `x -> expr` - функция `lambda(x, expr)` Следующие операторы не имеют приоритета, так как представляют собой скобки: -Оператор создания массива -------------------------- +## Оператор создания массива `[x1, ...]` - функция `array(x1, ...)` -Оператор создания кортежа -------------------------- +## Оператор создания кортежа `(x1, x2, ...)` - функция `tuple(x2, x2, ...)` -Ассоциативность ---------------- +## Ассоциативность Все бинарные операторы имеют левую ассоциативность. Например, `1 + 2 + 3` преобразуется в `plus(plus(1, 2), 3)`. Иногда это работает не так, как ожидается. Например, `SELECT 4 > 3 > 2` выдаст 0. diff --git a/docs/ru/query_language/clickhouse_local.md b/docs/ru/query_language/clickhouse_local.md index 1402e341d88..ee3f50dfeef 100644 --- a/docs/ru/query_language/clickhouse_local.md +++ b/docs/ru/query_language/clickhouse_local.md @@ -1,4 +1,3 @@ -Программа clickhouse-local -========================== +# Программа clickhouse-local Программа `clickhouse-local` позволяет выполнять быструю обработку локальных файлов, хранящих таблицы, не прибегая к развертыванию и настройке clickhouse-server ... diff --git a/docs/ru/query_language/index.md b/docs/ru/query_language/index.md index 7a529c7dc07..cd5f2638a35 100644 --- a/docs/ru/query_language/index.md +++ b/docs/ru/query_language/index.md @@ -1,5 +1,4 @@ -Язык запросов -============= +# Язык запросов ```eval_rst .. toctree:: diff --git a/docs/ru/query_language/queries.md b/docs/ru/query_language/queries.md index 79bf2afbdbc..acf50619d7d 100644 --- a/docs/ru/query_language/queries.md +++ b/docs/ru/query_language/queries.md @@ -1,7 +1,6 @@ -Запросы -======= +#Запросы -### CREATE DATABASE +## CREATE DATABASE Создание базы данных db_name ```sql @@ -11,7 +10,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name `База данных` - это просто директория для таблиц. Если написано `IF NOT EXISTS`, то запрос не будет возвращать ошибку, если база данных уже существует. -### CREATE TABLE +## CREATE TABLE Запрос `CREATE TABLE` может иметь несколько форм. ```sql @@ -43,8 +42,8 @@ CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name ENGINE = engine AS SELECT ... Во всех случаях, если указано `IF NOT EXISTS`, то запрос не будет возвращать ошибку, если таблица уже существует. В этом случае, запрос будет ничего не делать. -Значения по умолчанию ---------------------- +### Значения по умолчанию + В описании столбца, может быть указано выражение для значения по умолчанию, одного из следующих видов: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. @@ -80,8 +79,7 @@ CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name ENGINE = engine AS SELECT ... Отсутствует возможность задать значения по умолчанию для элементов вложенных структур данных. -Временные таблицы ------------------ +### Временные таблицы Во всех случаях, если указано `TEMPORARY`, то будет создана временная таблица. Временные таблицы обладают следующими особенностями: - временные таблицы исчезают после завершения сессии; в том числе, при обрыве соединения; @@ -106,7 +104,7 @@ CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE Локальная версия запроса в конечном итоге будет выполнена на каждом хосте кластера, даже если некоторые хосты в данный момент не доступны, гарантируется упорядоченность выполнения запросов в рамках одного хоста. Пока не поддерживаются `ALTER`-запросы для реплицированных таблиц. -### CREATE VIEW +## CREATE VIEW ```sql CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]name [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... @@ -154,7 +152,7 @@ SELECT a, b, c FROM (SELECT ...) Отсутствует отдельный запрос для удаления представлений. Чтобы удалить представление, следует использовать `DROP TABLE`. -### ATTACH +## ATTACH Запрос полностью аналогичен запросу `CREATE`, но - вместо слова `CREATE` используется слово `ATTACH`; - запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице в сервер. @@ -168,7 +166,7 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением системных таблиц, создание которых явно вписано в сервер). -### DROP +## DROP Запрос имеет два вида: `DROP DATABASE` и `DROP TABLE`. ```sql @@ -185,7 +183,7 @@ DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Удаляет таблицу. Если указано `IF EXISTS` - не выдавать ошибку, если таблица не существует или база данных не существует. -### DETACH +## DETACH Удаляет из сервера информацию о таблице name. Сервер перестаёт знать о существовании таблицы. ```sql @@ -197,7 +195,7 @@ DETACH TABLE [IF EXISTS] [db.]name Запроса `DETACH DATABASE` нет. -### RENAME +## RENAME Переименовывает одну или несколько таблиц. ```sql @@ -208,11 +206,10 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... -### ALTER +## ALTER Запрос `ALTER` поддерживается только для таблиц типа `*MergeTree`, а также `Merge` и `Distributed`. Запрос имеет несколько вариантов. -Манипуляции со столбцами ------------------------- +### Манипуляции со столбцами Изменение структуры таблицы. @@ -275,8 +272,7 @@ MODIFY COLUMN name [type] [default_expr] Запрос `ALTER` на изменение столбцов реплицируется. Соответствующие инструкции сохраняются в ZooKeeper, и затем каждая реплика их применяет. Все запросы `ALTER` выполняются в одном и том же порядке. Запрос ждёт выполнения соответствующих действий на всех репликах. Но при этом, запрос на изменение столбцов в реплицируемой таблице можно прервать, и все действия будут осуществлены асинхронно. -Манипуляции с партициями и кусками ----------------------------------- +### Манипуляции с партициями и кусками Работает только для таблиц семейства `MergeTree`. Существуют следующие виды операций: @@ -396,8 +392,7 @@ ALTER TABLE [db.]table FREEZE PARTITION 'name' Таким образом, данные из бэкапа будут добавлены в таблицу. Восстановление из бэкапа, так же, не требует остановки сервера. -Бэкапы и репликация -------------------- +### Бэкапы и репликация Репликация защищает от аппаратных сбоев. В случае, если на одной из реплик у вас исчезли все данные, то восстановление делается по инструкции в разделе "Восстановление после сбоя". @@ -423,8 +418,7 @@ ALTER TABLE [db.]table FETCH PARTITION 'name' FROM 'path-in-zookeeper' Запрос `ALTER ... FETCH PARTITION` не реплицируется. То есть, партиция будет скачана в директорию detached только на локальном сервере. Заметим, что если вы после этого добавите данные в таблицу с помощью запроса `ALTER TABLE ... ATTACH`, то данные будут добавлены на всех репликах (на одной из реплик будут добавлены из директории detached, а на других - загружены с соседних реплик). -Синхронность запросов ALTER ---------------------------- +### Синхронность запросов ALTER Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах. @@ -433,7 +427,7 @@ ALTER TABLE [db.]table FETCH PARTITION 'name' FROM 'path-in-zookeeper' -### SHOW DATABASES +## SHOW DATABASES ```sql SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] @@ -444,7 +438,7 @@ SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] Смотрите также раздел "Форматы". -### SHOW TABLES +## SHOW TABLES ```sql SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] @@ -458,7 +452,7 @@ SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] Запрос полностью аналогичен запросу: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]` Смотрите также раздел "Оператор LIKE". -### SHOW PROCESSLIST +## SHOW PROCESSLIST ```sql SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] @@ -490,7 +484,7 @@ SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" ``` -### SHOW CREATE TABLE +## SHOW CREATE TABLE ```sql SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] @@ -498,7 +492,7 @@ SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] Возвращает один столбец statement типа `String`, содержащий одно значение - запрос `CREATE`, с помощью которого создана указанная таблица. -### DESCRIBE TABLE +## DESCRIBE TABLE ```sql DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] @@ -508,7 +502,7 @@ DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] Вложенные структуры данных выводятся в "развёрнутом" виде. То есть, каждый столбец - по отдельности, с именем через точку. -### EXISTS +## EXISTS ```sql EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] @@ -516,7 +510,7 @@ EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] Возвращает один столбец типа `UInt8`, содержащий одно значение - `0`, если таблицы или БД не существует и `1`, если таблица в указанной БД существует. -### USE +## USE ```sql USE db @@ -526,7 +520,7 @@ USE db Текущая база данных используется для поиска таблиц, если база данных не указана в запросе явно через точку перед именем таблицы. При использовании HTTP протокола, запрос не может быть выполнен, так как понятия сессии не существует. -### SET +## SET ```sql SET param = value @@ -539,7 +533,7 @@ SET param = value При перезапуске сервера, теряются настройки, установленные с помощью `SET`. Установить настройки, которые переживут перезапуск сервера, можно только с помощью конфигурационного файла сервера. -### OPTIMIZE +## OPTIMIZE ```sql OPTIMIZE TABLE [db.]name [PARTITION partition] [FINAL] @@ -552,7 +546,7 @@ OPTIMIZE TABLE [db.]name [PARTITION partition] [FINAL] -### INSERT +## INSERT Добавление данных. @@ -593,8 +587,7 @@ INSERT INTO t FORMAT TabSeparated С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе "[Интерфейсы](../interfaces/index.md#interfaces)". -Вставка результатов `SELECT` -============================ +### Вставка результатов `SELECT` ```sql INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... @@ -607,8 +600,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... Не поддерживаются другие запросы на модификацию части данных: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. Вы можете удалять старые данные с помощью запроса `ALTER TABLE ... DROP PARTITION`. -Замечания о производительности -============================== +### Замечания о производительности `INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по месяцам. Если вы вставляете данные за разные месяцы вперемешку, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого: @@ -620,7 +612,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - Данные поступают в режиме реального времени. - Вы загружаете данные, которые как правило отсортированы по времени. -### SELECT +## SELECT Выборка данных. @@ -648,8 +640,7 @@ SELECT [DISTINCT] expr_list Если в запросе отсутствуют секции `DISTINCT`, `GROUP BY`, `ORDER BY`, подзапросы в `IN` и `JOIN`, то запрос будет обработан полностью потоково, с использованием O(1) количества оперативки. Иначе запрос может съесть много оперативки, если не указаны подходящие ограничения `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. Подробнее смотрите в разделе "Настройки". Присутствует возможность использовать внешнюю сортировку (с сохранением временных данных на диск) и внешнюю агрегацию. `Merge join` в системе нет. -Секция FROM ------------ +### Секция FROM Если секция FROM отсутствует, то данные будут читаться из таблицы `system.one`. Таблица system.one содержит ровно одну строку (то есть, эта таблица выполняет такую же роль, как таблица DUAL, которую можно найти в других СУБД). @@ -667,8 +658,7 @@ SELECT [DISTINCT] expr_list Модификатор FINAL может быть использован только при SELECT-е из таблицы типа CollapsingMergeTree. При указании FINAL, данные будут выбираться полностью "сколлапсированными". Стоит учитывать, что использование FINAL приводит к выбору кроме указанных в SELECT-е столбцов также столбцов, относящихся к первичному ключу. Также, запрос будет выполняться в один поток, и при выполнении запроса будет выполняться слияние данных. Это приводит к тому, что при использовании FINAL, запрос выполняется медленнее. В большинстве случаев, следует избегать использования FINAL. Подробнее смотрите раздел "Движок CollapsingMergeTree". -Секция SAMPLE -------------- +### Секция SAMPLE Секция SAMPLE позволяет выполнить запрос приближённо. Приближённое выполнение запроса поддерживается только таблицами типа MergeTree\* и только если при создании таблицы было указано выражение, по которому производится выборка (смотрите раздел "Движок MergeTree"). @@ -704,8 +694,7 @@ ORDER BY PageViews DESC LIMIT 1000 Например, выборка по идентификаторам посетителей, выберет из разных таблиц строки с одинаковым подмножеством всех возможных идентификаторов посетителей. Это позволяет использовать выборку в подзапросах в секции IN, а также при ручном сопоставлении результатов разных запросов с выборками. -Секция ARRAY JOIN ------------------ +### Секция ARRAY JOIN Позволяет выполнить JOIN с массивом или вложенной структурой данных. Смысл похож на функцию arrayJoin, но функциональность более широкая. @@ -957,8 +946,7 @@ ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num Соответствующее преобразование может выполняться как до секции WHERE/PREWHERE (если его результат нужен в этой секции), так и после выполнения WHERE/PREWHERE (чтобы уменьшить объём вычислений). -Секция JOIN ------------ +### Секция JOIN Обычный JOIN, не имеет отношения к ARRAY JOIN, который описан выше. @@ -1054,16 +1042,14 @@ LIMIT 10 Если JOIN необходим для соединения с таблицами измерений (dimension tables - сравнительно небольшие таблицы, которые содержат свойства измерений - например, имена для рекламных кампаний), то использование JOIN может быть не очень удобным из-за громоздкости синтаксиса, а также из-за того, что правая таблица читается заново при каждом запросе. Специально для таких случаев существует функциональность "Внешние словари", которую следует использовать вместо JOIN. Подробнее смотрите раздел "Внешние словари". -Секция WHERE ------------- +### Секция WHERE Секция WHERE, если есть, должна содержать выражение, имеющее тип UInt8. Обычно это какое-либо выражение с операторами сравнения и логическими операторами. Это выражение будет использовано для фильтрации данных перед всеми остальными преобразованиями. Выражение анализируется на возможность использования индексов, если индексы поддерживаются движком таблицы. -Секция PREWHERE ---------------- +### Секция PREWHERE Имеет такой же смысл, как и секция WHERE. Отличие состоит в том, какие данные читаются из таблицы. При использовании PREWHERE, из таблицы сначала читаются только столбцы, необходимые для выполнения PREWHERE. Затем читаются остальные столбцы, нужные для выполнения запроса, но из них только те блоки, в которых выражение в PREWHERE истинное. @@ -1080,8 +1066,7 @@ PREWHERE поддерживается только таблицами семей Если настройка optimize_move_to_prewhere выставлена в 1, то при отсутствии PREWHERE, система будет автоматически переносить части выражений из WHERE в PREWHERE согласно некоторой эвристике. -Секция GROUP BY ---------------- +### Секция GROUP BY Это одна из наиболее важных частей СУБД. @@ -1164,7 +1149,7 @@ GROUP BY вычисляет для каждого встретившегося Если после GROUP BY у вас есть ORDER BY с небольшим LIMIT, то на ORDER BY не будет тратиться существенного количества оперативки. Но если есть ORDER BY без LIMIT, то не забудьте включить внешнюю сортировку (`max_bytes_before_external_sort`). -#### Модификатор LIMIT N BY +### Секция LIMIT N BY LIMIT N BY COLUMNS выбирает топ N строк для каждой группы COLUMNS. LIMIT N BY не связан с LIMIT и они могут использоваться в одном запросе. Ключ для LIMIT N BY может содержать произвольное число колонок или выражений. @@ -1185,8 +1170,7 @@ LIMIT 100 Запрос выберет топ 5 рефереров для каждой пары `domain, device_type`, но не более 100 строк (`LIMIT n BY + LIMIT`). -Секция HAVING -------------- +### Секция HAVING Позволяет отфильтровать результат, полученный после GROUP BY, аналогично секции WHERE. WHERE и HAVING отличаются тем, что WHERE выполняется до агрегации (GROUP BY), а HAVING - после. @@ -1194,8 +1178,7 @@ WHERE и HAVING отличаются тем, что WHERE выполняется -Секция ORDER BY ---------------- +### Секция ORDER BY Секция ORDER BY содержит список выражений, к каждому из которых также может быть приписано DESC или ASC (направление сортировки). Если ничего не приписано - это аналогично приписыванию ASC. ASC - сортировка по возрастанию, DESC - сортировка по убыванию. Обозначение направления сортировки действует на одно выражение, а не на весь список. Пример: `ORDER BY Visits DESC, SearchPhrase` @@ -1216,16 +1199,14 @@ WHERE и HAVING отличаются тем, что WHERE выполняется Внешняя сортировка работает существенно менее эффективно, чем сортировка в оперативке. -Секция SELECT -------------- +### Секция SELECT После вычислений, соответствующих всем перечисленным выше секциям, производится вычисление выражений, указанных в секции SELECT. Вернее, вычисляются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. Сами агрегатные функции и то, что под ними, вычисляются при агрегации (GROUP BY). Эти выражения работают так, как будто применяются к отдельным строкам результата. -Секция DISTINCT ---------------- +### Секция DISTINCT Если указано DISTINCT, то из всех множеств полностью совпадающих строк результата, будет оставляться только одна строка. Результат выполнения будет таким же, как если указано GROUP BY по всем указанным полям в SELECT-е и не указаны агрегатные функции. Но имеется несколько отличий от GROUP BY: @@ -1236,8 +1217,7 @@ WHERE и HAVING отличаются тем, что WHERE выполняется DISTINCT не поддерживается, если в SELECT-е присутствует хотя бы один столбец типа массив. -Секция LIMIT ------------- +### Секция LIMIT LIMIT m позволяет выбрать из результата первые m строк. LIMIT n, m позволяет выбрать из результата первые m строк после пропуска первых n строк. @@ -1246,8 +1226,7 @@ n и m должны быть неотрицательными целыми чи При отсутствии секции ORDER BY, однозначно сортирующей результат, результат может быть произвольным и может являться недетерминированным. -Секция UNION ALL ----------------- +### Секция UNION ALL Произвольное количество запросов может быть объединено с помощью UNION ALL. Пример: @@ -1272,8 +1251,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Запросы - части UNION ALL нельзя заключить в скобки. ORDER BY и LIMIT применяются к отдельным запросам, а не к общему результату. Если вам нужно применить какое-либо преобразование к общему результату, то вы можете разместить все запросы с UNION ALL в подзапросе в секции FROM. -Секция INTO OUTFILE -------------------- +### Секция INTO OUTFILE При указании `INTO OUTFILE filename` (где filename - строковый литерал), результат запроса будет сохранён в файл filename. В отличие от MySQL, файл создаётся на стороне клиента. Если файл с таким именем уже существует, это приведёт к ошибке. @@ -1281,8 +1259,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c Формат вывода по умолчанию - TabSeparated, как и в неинтерактивном режиме клиента командной строки. -Секция FORMAT -------------- +### Секция FORMAT При указании FORMAT format вы можете получить данные в любом указанном формате. Это может использоваться для удобства или для создания дампов. @@ -1291,8 +1268,7 @@ SELECT CounterID, 2 AS table, sum(Sign) AS c При использовании клиента командной строки данные на клиент передаются во внутреннем эффективном формате. При этом клиент самостоятельно интерпретирует секцию FORMAT запроса и форматирует данные на своей стороне (снимая нагрузку на сеть и сервер). -Операторы IN ------------- +### Операторы IN Операторы `IN`, `NOT IN`, `GLOBAL IN`, `GLOBAL NOT IN` рассматриваются отдельно, так как их функциональность достаточно богатая. @@ -1358,8 +1334,7 @@ ORDER BY EventDate ASC -Распределённые подзапросы -------------------------- +#### Распределённые подзапросы Существует два варианта IN-ов с подзапросами (аналогично для JOIN-ов): обычный `IN` / `JOIN` и `GLOBAL IN` / `GLOBAL JOIN`. Они отличаются способом выполнения при распределённой обработке запроса. @@ -1463,8 +1438,7 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL В секции `GLOBAL IN` также имеет смысл указывать локальную таблицу - в случае, если эта локальная таблица есть только на сервере-инициаторе запроса, и вы хотите воспользоваться данными из неё на удалённых серверах. -Экстремальные значения ----------------------- +### Экстремальные значения Вы можете получить в дополнение к результату также минимальные и максимальные значения по столбцам результата. Для этого выставите настройку **extremes** в 1. Минимумы и максимумы считаются для числовых типов, дат, дат-с-временем. Для остальных столбцов будут выведены значения по умолчанию. @@ -1474,8 +1448,7 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL Экстремальные значения считаются по строчкам, прошедшим через LIMIT. Но при этом, при использовании LIMIT offset, size, строчки до offset учитываются в extremes. В потоковых запросах, в результате может учитываться также небольшое количество строчек, прошедших LIMIT. -Замечания ---------- +### Замечания В секциях `GROUP BY`, `ORDER BY`, в отличие от диалекта MySQL, и в соответствии со стандартным SQL, не поддерживаются позиционные аргументы. Например, если вы напишите `GROUP BY 1, 2` - то это будет воспринято, как группировка по константам (то есть, агрегация всех строк в одну). @@ -1492,7 +1465,7 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL В других случаях использование звёздочки является издевательством над системой, так как вместо преимуществ столбцовой СУБД вы получаете недостатки. То есть использовать звёздочку не рекомендуется. -### KILL QUERY +## KILL QUERY ```sql KILL QUERY WHERE [SYNC|ASYNC|TEST] [FORMAT format] diff --git a/docs/ru/query_language/syntax.md b/docs/ru/query_language/syntax.md index af97ba99a03..11752aaed5f 100644 --- a/docs/ru/query_language/syntax.md +++ b/docs/ru/query_language/syntax.md @@ -1,5 +1,4 @@ -Синтаксис -========= +# Синтаксис В системе есть два вида парсеров: полноценный парсер SQL (recursive descent parser) и парсер форматов данных (быстрый потоковый парсер). Во всех случаях кроме запроса INSERT, используется только полноценный парсер SQL. @@ -17,33 +16,28 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') Далее пойдёт речь о полноценном парсере. О парсерах форматов, смотри раздел "Форматы". -Пробелы -------- +## Пробелы Между синтаксическими конструкциями (в том числе, в начале и конце запроса) может быть расположено произвольное количество пробельных символов. К пробельным символам относятся пробел, таб, перевод строки, CR, form feed. -Комментарии ------------ +## Комментарии Поддерживаются комментарии в SQL-стиле и C-стиле. Комментарии в SQL-стиле: от `--` до конца строки. Пробел после `--` может не ставиться. Комментарии в C-стиле: от `/*` до `*/`. Такие комментарии могут быть многострочными. Пробелы тоже не обязательны. -Ключевые слова --------------- +## Ключевые слова Ключевые слова (например, `SELECT`) регистронезависимы. Всё остальное (имена столбцов, функций и т. п.), в отличие от стандарта SQL, регистрозависимо. Ключевые слова не зарезервированы (а всего лишь парсятся как ключевые слова в соответствующем контексте). -Идентификаторы --------------- +## Идентификаторы Идентификаторы (имена столбцов, функций, типов данных) могут быть квотированными или не квотированными. Не квотированные идентификаторы начинаются на букву латинского алфавита или подчёркивание; продолжаются на букву латинского алфавита или подчёркивание или цифру. Короче говоря, должны соответствовать регулярному выражению `^[a-zA-Z_][0-9a-zA-Z_]*$`. Примеры: `x, _1, X_y__Z123_.` Квотированные идентификаторы расположены в обратных кавычках `` `id ``\` (также, как в MySQL), и могут обозначать произвольный (непустой) набор байт. При этом, внутри записи такого идентификатора, символы (например, символ обратной кавычки) могут экранироваться с помощью обратного слеша. Правила экранирования такие же, как в строковых литералах (см. ниже). Рекомендуется использовать идентификаторы, которые не нужно квотировать. -Литералы --------- +## Литералы Бывают числовые, строковые и составные литералы. @@ -74,26 +68,22 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') Массив должен состоять хотя бы из одного элемента, а кортеж - хотя бы из двух. Кортежи носят служебное значение для использования в секции IN запроса SELECT. Кортежи могут быть получены в качестве результата запроса, но не могут быть сохранены в базу (за исключением таблиц типа Memory). -Функции -------- +## Функции Функции записываются как идентификатор со списком аргументов (возможно, пустым) в скобках. В отличие от стандартного SQL, даже в случае пустого списка аргументов, скобки обязательны. Пример: `now()`. Бывают обычные и агрегатные функции (смотрите раздел "Агрегатные функции"). Некоторые агрегатные функции могут содержать два списка аргументов в круглых скобках. Пример: `quantile(0.9)(x)`. Такие агрегатные функции называются "параметрическими", а первый список аргументов называется "параметрами". Синтаксис агрегатных функций без параметров ничем не отличается от обычных функций. -Операторы ---------- +## Операторы Операторы преобразуются в соответствующие им функции во время парсинга запроса, с учётом их приоритета и ассоциативности. Например, выражение `1 + 2 * 3 + 4` преобразуется в `plus(plus(1, multiply(2, 3)), 4)`. Подробнее смотрите раздел "Операторы" ниже. -Типы данных и движки таблиц ---------------------------- +## Типы данных и движки таблиц Типы данных и движки таблиц в запросе `CREATE` записываются также, как идентификаторы или также как функции. То есть, могут содержать или не содержать список аргументов в круглых скобках. Подробнее смотрите разделы "Типы данных", "Движки таблиц", "CREATE". -Синонимы --------- +## Синонимы В запросе SELECT, в выражениях могут быть указаны синонимы с помощью ключевого слова AS. Слева от AS стоит любое выражение. Справа от AS стоит идентификатор - имя для синонима. В отличие от стандартного SQL, синонимы могут объявляться не только на верхнем уровне выражений: @@ -103,13 +93,11 @@ SELECT (1 AS n) + 2, n В отличие от стандартного SQL, синонимы могут использоваться во всех секциях запроса, а не только `SELECT`. -Звёздочка ---------- +## Звёздочка В запросе `SELECT`, вместо выражения может стоять звёздочка. Подробнее смотрите раздел "SELECT". -Выражения ---------- +## Выражения Выражение представляет собой функцию, идентификатор, литерал, применение оператора, выражение в скобках, подзапрос, звёздочку; и может содержать синоним. Список выражений - одно выражение или несколько выражений через запятую. diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md index 4da0b1bb161..1443af70320 100644 --- a/docs/ru/roadmap.md +++ b/docs/ru/roadmap.md @@ -1,23 +1,19 @@ -Roadmap -======= +# Roadmap -Q3 2017 -------- +## Q3 2017 - Запросы `SYSTEM` - Ограничение числа параллельных скачиваний с реплик - Доделать поддержку `NULL` - `SELECT db.table.column` -Q4 2017 -------- +## Q4 2017 - Произвольный ключ партиционирования для движков семейства MergeTree - Доработки синтаксиса `JOIN` для совместимости со стандартом SQL - Пулы ресурсов для запросов (CPU, disk I/O, network bandwidth) -Q1 2018 -------- +## Q1 2018 - Начальная поддержка `UPDATE` и `DELETE` diff --git a/docs/ru/system_tables/index.md b/docs/ru/system_tables/index.md index 2a7d952ee0b..26fa3b8e60a 100644 --- a/docs/ru/system_tables/index.md +++ b/docs/ru/system_tables/index.md @@ -1,5 +1,4 @@ -Системные таблицы -================= +# Системные таблицы Системные таблицы используются для реализации части функциональности системы, а также предоставляют доступ к информации о работе системы. Вы не можете удалить системную таблицу (хотя можете сделать DETACH). diff --git a/docs/ru/system_tables/system.asynchronous_metrics.md b/docs/ru/system_tables/system.asynchronous_metrics.md index b449070cdf0..b1e649ac16a 100644 --- a/docs/ru/system_tables/system.asynchronous_metrics.md +++ b/docs/ru/system_tables/system.asynchronous_metrics.md @@ -1,7 +1,6 @@ -system.asynchronous_metrics -============================ +# system.asynchronous_metrics Содержат метрики, используемые для профилирования и мониторинга. Обычно отражают количество событий, происходящих в данный момент в системе, или ресурсов, суммарно потребляемых системой. diff --git a/docs/ru/system_tables/system.clusters.md b/docs/ru/system_tables/system.clusters.md index b00cea45fb9..52116a127cb 100644 --- a/docs/ru/system_tables/system.clusters.md +++ b/docs/ru/system_tables/system.clusters.md @@ -1,5 +1,4 @@ -system.clusters -=============== +# system.clusters Содержит информацию о доступных в конфигурационном файле кластерах и серверах, которые в них входят. Столбцы: diff --git a/docs/ru/system_tables/system.columns.md b/docs/ru/system_tables/system.columns.md index 26102e47349..c458f24d95f 100644 --- a/docs/ru/system_tables/system.columns.md +++ b/docs/ru/system_tables/system.columns.md @@ -1,5 +1,4 @@ -system.columns -============== +# system.columns Содержит информацию о столбцах всех таблиц. С помощью этой таблицы можно получить информацию аналогично запросу `DESCRIBE TABLE`, но для многих таблиц сразу. diff --git a/docs/ru/system_tables/system.databases.md b/docs/ru/system_tables/system.databases.md index 91585ce33df..10087837b5c 100644 --- a/docs/ru/system_tables/system.databases.md +++ b/docs/ru/system_tables/system.databases.md @@ -1,5 +1,4 @@ -system.databases -================ +# system.databases Таблица содержит один столбец name типа String - имя базы данных. Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. diff --git a/docs/ru/system_tables/system.dictionaries.md b/docs/ru/system_tables/system.dictionaries.md index 73390be5b6d..67b1af8c6b4 100644 --- a/docs/ru/system_tables/system.dictionaries.md +++ b/docs/ru/system_tables/system.dictionaries.md @@ -1,5 +1,4 @@ -system.dictionaries -=================== +# system.dictionaries Содержит информацию о внешних словарях. diff --git a/docs/ru/system_tables/system.events.md b/docs/ru/system_tables/system.events.md index 4d2d84f8082..89ced73459c 100644 --- a/docs/ru/system_tables/system.events.md +++ b/docs/ru/system_tables/system.events.md @@ -1,7 +1,6 @@ -system.events -============= +# system.events Содержит информацию о количестве произошедших в системе событий, для профилирования и мониторинга. Пример: количество обработанных запросов типа SELECT. diff --git a/docs/ru/system_tables/system.functions.md b/docs/ru/system_tables/system.functions.md index 280a846683d..0f96a6fa167 100644 --- a/docs/ru/system_tables/system.functions.md +++ b/docs/ru/system_tables/system.functions.md @@ -1,5 +1,4 @@ -system.functions -================ +# system.functions Содержит информацию об обычных и агрегатных функциях. diff --git a/docs/ru/system_tables/system.merges.md b/docs/ru/system_tables/system.merges.md index 04bc5b10f2e..07439e04d75 100644 --- a/docs/ru/system_tables/system.merges.md +++ b/docs/ru/system_tables/system.merges.md @@ -1,5 +1,4 @@ -system.merges -============= +# system.merges Содержит информацию о производящихся прямо сейчас слияниях для таблиц семейства MergeTree. diff --git a/docs/ru/system_tables/system.metrics.md b/docs/ru/system_tables/system.metrics.md index ab2138f9f3b..3fdc8b0daee 100644 --- a/docs/ru/system_tables/system.metrics.md +++ b/docs/ru/system_tables/system.metrics.md @@ -1,4 +1,3 @@ -system.metrics -============== +# system.metrics diff --git a/docs/ru/system_tables/system.numbers.md b/docs/ru/system_tables/system.numbers.md index 7eb15adf551..6a7ebc78cf5 100644 --- a/docs/ru/system_tables/system.numbers.md +++ b/docs/ru/system_tables/system.numbers.md @@ -1,5 +1,4 @@ -system.numbers -============== +# system.numbers Таблица содержит один столбец с именем number типа UInt64, содержащим почти все натуральные числа, начиная с нуля. Эту таблицу можно использовать для тестов, а также если вам нужно сделать перебор. diff --git a/docs/ru/system_tables/system.numbers_mt.md b/docs/ru/system_tables/system.numbers_mt.md index 4017f3a4452..113abbf4a26 100644 --- a/docs/ru/system_tables/system.numbers_mt.md +++ b/docs/ru/system_tables/system.numbers_mt.md @@ -1,5 +1,4 @@ -system.numbers_mt -================== +# system.numbers_mt То же самое, что и system.numbers, но чтение распараллеливается. Числа могут возвращаться в произвольном порядке. Используется для тестов. diff --git a/docs/ru/system_tables/system.one.md b/docs/ru/system_tables/system.one.md index cf97fa38c38..b3666757e3a 100644 --- a/docs/ru/system_tables/system.one.md +++ b/docs/ru/system_tables/system.one.md @@ -1,5 +1,4 @@ -system.one -========== +# system.one Таблица содержит одну строку с одним столбцом dummy типа UInt8, содержащим значение 0. Эта таблица используется, если в SELECT запросе не указана секция FROM. diff --git a/docs/ru/system_tables/system.parts.md b/docs/ru/system_tables/system.parts.md index 4475958e491..8a5a0f34576 100644 --- a/docs/ru/system_tables/system.parts.md +++ b/docs/ru/system_tables/system.parts.md @@ -1,5 +1,4 @@ -system.parts -============ +# system.parts Содержит информацию о кусках таблиц семейства [MergeTree](../table_engines/mergetree.md#table_engines-mergetree). diff --git a/docs/ru/system_tables/system.processes.md b/docs/ru/system_tables/system.processes.md index c4680009209..82264c0b794 100644 --- a/docs/ru/system_tables/system.processes.md +++ b/docs/ru/system_tables/system.processes.md @@ -1,5 +1,4 @@ -system.processes -================ +# system.processes Эта системная таблица используется для реализации запроса `SHOW PROCESSLIST`. Столбцы: diff --git a/docs/ru/system_tables/system.replicas.md b/docs/ru/system_tables/system.replicas.md index 2b52e20f514..5647dd24edb 100644 --- a/docs/ru/system_tables/system.replicas.md +++ b/docs/ru/system_tables/system.replicas.md @@ -1,5 +1,4 @@ -system.replicas -=============== +# system.replicas Содержит информацию и статус для реплицируемых таблиц, расположенных на локальном сервере. Эту таблицу можно использовать для мониторинга. Таблица содержит по строчке для каждой Replicated\*-таблицы. diff --git a/docs/ru/system_tables/system.settings.md b/docs/ru/system_tables/system.settings.md index 2884b5dbefd..6fd12cd39fd 100644 --- a/docs/ru/system_tables/system.settings.md +++ b/docs/ru/system_tables/system.settings.md @@ -1,5 +1,4 @@ -system.settings -=============== +# system.settings Содержит информацию о настройках, используемых в данный момент. То есть, используемых для выполнения запроса, с помощью которого вы читаете из таблицы system.settings. diff --git a/docs/ru/system_tables/system.tables.md b/docs/ru/system_tables/system.tables.md index cf5909427a3..4e346a72228 100644 --- a/docs/ru/system_tables/system.tables.md +++ b/docs/ru/system_tables/system.tables.md @@ -1,5 +1,4 @@ -system.tables -============= +# system.tables Таблица содержит столбцы database, name, engine типа String и столбец metadata_modification_time типа DateTime. Для каждой таблицы, о которой знает сервер, будет присутствовать соответствующая запись в таблице system.tables. diff --git a/docs/ru/system_tables/system.zookeeper.md b/docs/ru/system_tables/system.zookeeper.md index 31d8dbb2b35..5753c6166db 100644 --- a/docs/ru/system_tables/system.zookeeper.md +++ b/docs/ru/system_tables/system.zookeeper.md @@ -1,5 +1,4 @@ -system.zookeeper -================ +# system.zookeeper Позволяет читать данные из ZooKeeper кластера, описанного в конфигурации. В запросе обязательно в секции WHERE должно присутствовать условие на равенство path - путь в ZooKeeper, для детей которого вы хотите получить данные. diff --git a/docs/ru/table_engines/aggregatingmergetree.md b/docs/ru/table_engines/aggregatingmergetree.md index 2ae86ad3c78..48eee7e8e1c 100644 --- a/docs/ru/table_engines/aggregatingmergetree.md +++ b/docs/ru/table_engines/aggregatingmergetree.md @@ -1,5 +1,4 @@ -AggregatingMergeTree -==================== +# AggregatingMergeTree Отличается от MergeTree тем, что при слиянии, выполняет объединение состояний агрегатных функций, хранимых в таблице, для строчек с одинаковым значением первичного ключа. diff --git a/docs/ru/table_engines/buffer.md b/docs/ru/table_engines/buffer.md index be8fa39319d..7f947ea91b6 100644 --- a/docs/ru/table_engines/buffer.md +++ b/docs/ru/table_engines/buffer.md @@ -1,5 +1,4 @@ -Buffer -====== +# Buffer Буферизует записываемые данные в оперативке, периодически сбрасывая их в другую таблицу. При чтении, производится чтение данных одновременно из буфера и из другой таблицы. diff --git a/docs/ru/table_engines/collapsingmergetree.md b/docs/ru/table_engines/collapsingmergetree.md index 88a82394bff..6336c0d3000 100644 --- a/docs/ru/table_engines/collapsingmergetree.md +++ b/docs/ru/table_engines/collapsingmergetree.md @@ -1,5 +1,4 @@ -CollapsingMergeTree -=================== +# CollapsingMergeTree *Движок достаточно специфичен для Яндекс.Метрики.* diff --git a/docs/ru/table_engines/custom_partitioning_key.md b/docs/ru/table_engines/custom_partitioning_key.md index 62572668e4b..f72900d1ac6 100644 --- a/docs/ru/table_engines/custom_partitioning_key.md +++ b/docs/ru/table_engines/custom_partitioning_key.md @@ -1,7 +1,6 @@ -Произвольный ключ партиционирования -=================================== +# Произвольный ключ партиционирования Начиная с версии 1.1.54310 доступна возможность создания таблиц семейства MergeTree с произвольным выражением партиционирования (не только по месяцу). diff --git a/docs/ru/table_engines/distributed.md b/docs/ru/table_engines/distributed.md index 43c8ed16134..a28ebb115cd 100644 --- a/docs/ru/table_engines/distributed.md +++ b/docs/ru/table_engines/distributed.md @@ -1,7 +1,6 @@ -Distributed -=========== +# Distributed **Движок Distributed не хранит данные самостоятельно**, а позволяет обрабатывать запросы распределённо, на нескольких серверах. Чтение автоматически распараллеливается. При чтении будут использованы индексы таблиц на удалённых серверах, если есть. diff --git a/docs/ru/table_engines/external_data.md b/docs/ru/table_engines/external_data.md index 42ea8ac4ce5..fc61c77677c 100644 --- a/docs/ru/table_engines/external_data.md +++ b/docs/ru/table_engines/external_data.md @@ -1,5 +1,4 @@ -Внешние данные для обработки запроса -==================================== +# Внешние данные для обработки запроса ClickHouse позволяет отправить на сервер данные, необходимые для обработки одного запроса, вместе с запросом SELECT. Такие данные будут положены во временную таблицу (см. раздел "Временные таблицы") и смогут использоваться в запросе (например, в операторах IN). diff --git a/docs/ru/table_engines/file.md b/docs/ru/table_engines/file.md index 158b930b4be..e9c221c56a5 100644 --- a/docs/ru/table_engines/file.md +++ b/docs/ru/table_engines/file.md @@ -1,4 +1,3 @@ -File(InputFormat) -================= +# File(InputFormat) Источником данных является файл, хранящий данные в одном из поддерживаемых форматов входных данных (TabSeparated, Native, и т. д.) ... diff --git a/docs/ru/table_engines/graphitemergetree.md b/docs/ru/table_engines/graphitemergetree.md index 3211873fd30..f3dad938c34 100644 --- a/docs/ru/table_engines/graphitemergetree.md +++ b/docs/ru/table_engines/graphitemergetree.md @@ -1,7 +1,6 @@ -GraphiteMergeTree -================= +# GraphiteMergeTree Движок предназначен для rollup (прореживания и агрегирования/усреднения) данных [Graphite](http://graphite.readthedocs.io/en/latest/index.html). Он может быть интересен разработчикам, которые хотят использовать ClickHouse как хранилище данных для Graphite. @@ -17,8 +16,7 @@ Graphite хранит в ClickHouse полные данные, а получат Движок наследует свойства MergeTree. Настройки прореживания данных задаются параметром [graphite_rollup](../operations/server_settings/settings.md#server_settings-graphite_rollup) в конфигурации сервера . -Использование движка --------------------- +## Использование движка Таблица с данными Graphite должна содержать как минимум следующие поля: diff --git a/docs/ru/table_engines/index.md b/docs/ru/table_engines/index.md index f1a95d2e407..6ff5a065570 100644 --- a/docs/ru/table_engines/index.md +++ b/docs/ru/table_engines/index.md @@ -1,5 +1,4 @@ -Движки таблиц -============= +# Движки таблиц Движок таблицы (тип таблицы) определяет: diff --git a/docs/ru/table_engines/join.md b/docs/ru/table_engines/join.md index 6f3637b06ad..2acabefc491 100644 --- a/docs/ru/table_engines/join.md +++ b/docs/ru/table_engines/join.md @@ -1,5 +1,4 @@ -Join -==== +# Join Представляет собой подготовленную структуру данных для JOIN-а, постоянно находящуюся в оперативке. diff --git a/docs/ru/table_engines/log.md b/docs/ru/table_engines/log.md index 2ac1909711c..cfb050f1640 100644 --- a/docs/ru/table_engines/log.md +++ b/docs/ru/table_engines/log.md @@ -1,5 +1,4 @@ -Log -=== +# Log Отличается от TinyLog тем, что вместе с файлами столбцов лежит небольшой файл "засечек". Засечки пишутся на каждый блок данных и содержат смещение - с какого места нужно читать файл, чтобы пропустить заданное количество строк. Это позволяет читать данные из таблицы в несколько потоков. При конкуррентном доступе к данным, чтения могут выполняться одновременно, а записи блокируют чтения и друг друга. diff --git a/docs/ru/table_engines/materializedview.md b/docs/ru/table_engines/materializedview.md index f012ad4176d..1198fd18369 100644 --- a/docs/ru/table_engines/materializedview.md +++ b/docs/ru/table_engines/materializedview.md @@ -1,4 +1,3 @@ -MaterializedView -================ +# MaterializedView Используется для реализации материализованных представлений (подробнее см. запрос `CREATE MATERIALIZED VIEW`). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. diff --git a/docs/ru/table_engines/memory.md b/docs/ru/table_engines/memory.md index 57aed94168b..29f195ae76d 100644 --- a/docs/ru/table_engines/memory.md +++ b/docs/ru/table_engines/memory.md @@ -1,5 +1,4 @@ -Memory -====== +# Memory Хранит данные в оперативке, в несжатом виде. Данные хранятся именно в таком виде, в каком они получаются при чтении. То есть, само чтение из этой таблицы полностью бесплатно. Конкуррентный доступ к данным синхронизируется. Блокировки короткие: чтения и записи не блокируют друг друга. diff --git a/docs/ru/table_engines/merge.md b/docs/ru/table_engines/merge.md index 8c87f9b5629..b3d9ced8f89 100644 --- a/docs/ru/table_engines/merge.md +++ b/docs/ru/table_engines/merge.md @@ -1,5 +1,4 @@ -Merge -===== +# Merge Движок Merge (не путайте с движком `MergeTree`) не хранит данные самостоятельно, а позволяет читать одновременно из произвольного количества других таблиц. Чтение автоматически распараллеливается. Запись в таблицу не поддерживается. При чтении будут использованы индексы тех таблиц, из которых реально идёт чтение, если они существуют. @@ -21,8 +20,7 @@ Merge(hits, '^WatchLog') Типичный способ использования движка Merge - возможность работы с большим количеством таблиц типа TinyLog, как с одной. -Виртуальные столбцы -------------------- +## Виртуальные столбцы Виртуальные столбцы - столбцы, предоставляемые движком таблиц, независимо от определения таблицы. То есть, такие столбцы не указываются в CREATE TABLE, но доступны для SELECT-а. diff --git a/docs/ru/table_engines/mergetree.md b/docs/ru/table_engines/mergetree.md index ccc58591307..4da83967e26 100644 --- a/docs/ru/table_engines/mergetree.md +++ b/docs/ru/table_engines/mergetree.md @@ -1,7 +1,6 @@ -MergeTree -========= +# MergeTree Движок MergeTree поддерживает индекс по первичному ключу и по дате, и обеспечивает возможность обновления данных в реальном времени. Это наиболее продвинутый движок таблиц в ClickHouse. Не путайте с движком Merge. diff --git a/docs/ru/table_engines/null.md b/docs/ru/table_engines/null.md index a296a7226c5..252517b1acd 100644 --- a/docs/ru/table_engines/null.md +++ b/docs/ru/table_engines/null.md @@ -1,5 +1,4 @@ -Null -==== +# Null При записи в таблицу типа Null, данные игнорируются. При чтении из таблицы типа Null, возвращается пустота. diff --git a/docs/ru/table_engines/replacingmergetree.md b/docs/ru/table_engines/replacingmergetree.md index a345dc74600..4c86b489f99 100644 --- a/docs/ru/table_engines/replacingmergetree.md +++ b/docs/ru/table_engines/replacingmergetree.md @@ -1,5 +1,4 @@ -ReplacingMergeTree -================== +# ReplacingMergeTree Движок таблиц отличается от `MergeTree` тем, что выполняет удаление дублирующихся записей с одинаковым значением первичного ключа. diff --git a/docs/ru/table_engines/replication.md b/docs/ru/table_engines/replication.md index f479771542d..e0237eb2ed9 100644 --- a/docs/ru/table_engines/replication.md +++ b/docs/ru/table_engines/replication.md @@ -1,15 +1,14 @@ -Репликация данных -================= +# Репликация данных -### ReplicatedMergeTree +## ReplicatedMergeTree -### ReplicatedCollapsingMergeTree +## ReplicatedCollapsingMergeTree -### ReplicatedAggregatingMergeTree +## ReplicatedAggregatingMergeTree -### ReplicatedSummingMergeTree +## ReplicatedSummingMergeTree Репликация поддерживается только для таблиц семейства MergeTree. Репликация работает на уровне отдельных таблиц, а не всего сервера. То есть, на сервере могут быть расположены одновременно реплицируемые и не реплицируемые таблицы. @@ -65,8 +64,7 @@ -Создание реплицируемых таблиц ------------------------------ +## Создание реплицируемых таблиц В начало имени движка таблицы добавляется `Replicated`. Например, `ReplicatedMergeTree`. @@ -107,8 +105,7 @@ ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', Even Для удаления реплики, выполните запрос DROP TABLE. При этом, удаляется только одна реплика - расположенная на том сервере, где вы выполняете запрос. -Восстановление после сбоя -------------------------- +## Восстановление после сбоя Если при старте сервера, недоступен ZooKeeper, реплицируемые таблицы переходят в режим только для чтения. Система будет пытаться периодически установить соединение с ZooKeeper. @@ -132,8 +129,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data Затем запустите сервер. При старте, сервер удалит эти флаги и запустит восстановление. -Восстановление в случае потери всех данных ------------------------------------------- +## Восстановление в случае потери всех данных Если на одном из серверов исчезли все данные и метаданные, восстановление делается следующим образом: @@ -148,8 +144,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data Отсутствует ограничение на использование сетевой полосы при восстановлении. Имейте это ввиду, если восстанавливаете сразу много реплик. -Преобразование из MergeTree в ReplicatedMergeTree -------------------------------------------------- +## Преобразование из MergeTree в ReplicatedMergeTree Здесь и далее, под `MergeTree` подразумеваются все движки таблиц семейства `MergeTree`, так же для `ReplicatedMergeTree`. @@ -161,8 +156,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data Перенесите данные из старой таблицы в поддиректорию detached в директории с данными новой таблицы (`/var/lib/clickhouse/data/db_name/table_name/`). Затем добавьте эти куски данных в рабочий набор с помощью выполнения запросов ALTER TABLE ATTACH PARTITION на одной из реплик. -Преобразование из ReplicatedMergeTree в MergeTree -------------------------------------------------- +## Преобразование из ReplicatedMergeTree в MergeTree Создайте таблицу типа MergeTree с другим именем. Перенесите в её директорию с данными все данные из директории с данными таблицы типа ReplicatedMergeTree. Затем удалите таблицу типа ReplicatedMergeTree и перезапустите сервер. @@ -173,8 +167,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data После этого, вы можете запустить сервер, создать таблицу типа MergeTree, перенести данные в её директорию, и перезапустить сервер. -Восстановление в случае потери или повреждения метаданных на ZooKeeper кластере -------------------------------------------------------------------------------- +## Восстановление в случае потери или повреждения метаданных на ZooKeeper кластере Если данные в ZooKeeper оказались утеряны или повреждены, то вы можете сохранить данные, переместив их в нереплицируемую таблицу, как описано в пункте выше. diff --git a/docs/ru/table_engines/set.md b/docs/ru/table_engines/set.md index e4c8eb1d9b3..0697b32b492 100644 --- a/docs/ru/table_engines/set.md +++ b/docs/ru/table_engines/set.md @@ -1,5 +1,4 @@ -Set -=== +# Set Представляет собой множество, постоянно находящееся в оперативке. Предназначено для использования в правой части оператора IN (смотрите раздел "Операторы IN"). diff --git a/docs/ru/table_engines/summingmergetree.md b/docs/ru/table_engines/summingmergetree.md index 4f0f8aea8e0..da49ae0b257 100644 --- a/docs/ru/table_engines/summingmergetree.md +++ b/docs/ru/table_engines/summingmergetree.md @@ -1,5 +1,4 @@ -SummingMergeTree -================ +# SummingMergeTree Отличается от `MergeTree` тем, что суммирует данные при слиянии. diff --git a/docs/ru/table_engines/tinylog.md b/docs/ru/table_engines/tinylog.md index e8db4e4717f..ce4ddac68f5 100644 --- a/docs/ru/table_engines/tinylog.md +++ b/docs/ru/table_engines/tinylog.md @@ -1,5 +1,4 @@ -TinyLog -======= +# TinyLog Самый простой движок таблиц, который хранит данные на диске. Каждый столбец хранится в отдельном сжатом файле. diff --git a/docs/ru/table_engines/view.md b/docs/ru/table_engines/view.md index ba3008e1e14..128986eb733 100644 --- a/docs/ru/table_engines/view.md +++ b/docs/ru/table_engines/view.md @@ -1,4 +1,3 @@ -View -==== +# View Используется для реализации представлений (подробнее см. запрос `CREATE VIEW`). Не хранит данные, а хранит только указанный запрос `SELECT`. При чтении из таблицы, выполняет его (с удалением из запроса всех ненужных столбцов). diff --git a/docs/ru/table_functions/index.md b/docs/ru/table_functions/index.md index 30e2dc293fe..16c9ab69618 100644 --- a/docs/ru/table_functions/index.md +++ b/docs/ru/table_functions/index.md @@ -1,5 +1,4 @@ -Табличные функции -================= +# Табличные функции Табличные функции могут указываться в секции FROM вместо имени БД и таблицы. Табличные функции можно использовать только если не выставлена настройка readonly. diff --git a/docs/ru/table_functions/merge.md b/docs/ru/table_functions/merge.md index ff8543dd503..092c9243fc5 100644 --- a/docs/ru/table_functions/merge.md +++ b/docs/ru/table_functions/merge.md @@ -1,5 +1,4 @@ -merge -===== +# merge `merge(db_name, 'tables_regexp')` - создаёт временную таблицу типа Merge. Подробнее смотрите раздел "Движки таблиц, Merge". diff --git a/docs/ru/table_functions/remote.md b/docs/ru/table_functions/remote.md index 95f71fd89d3..ea62b52a0a9 100644 --- a/docs/ru/table_functions/remote.md +++ b/docs/ru/table_functions/remote.md @@ -1,7 +1,6 @@ -remote -====== +# remote Позволяет обратиться к удалённым серверам без создания таблицы типа `Distributed`. From 68b2f066360920da4fc1a077b69c4159d79335b9 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Dec 2017 17:27:52 +0300 Subject: [PATCH 2/3] Removed server setting belongs to resharding. Some links are fixed. --- docs/ru/formats/capnproto.md | 52 +++--- docs/ru/operations/access_rights.md | 2 +- .../ru/operations/server_settings/settings.md | 18 +-- docs/ru/operations/settings/settings.md | 2 +- docs/ru/table_engines/kafka.md | 152 +++++++++--------- 5 files changed, 105 insertions(+), 121 deletions(-) diff --git a/docs/ru/formats/capnproto.md b/docs/ru/formats/capnproto.md index 3364b1affc1..f3a309434f4 100644 --- a/docs/ru/formats/capnproto.md +++ b/docs/ru/formats/capnproto.md @@ -1,26 +1,26 @@ - - -# CapnProto - -Cap'n Proto - формат бинарных сообщений, похож на Protocol Buffers и Thrift, но не похож на JSON или MessagePack. - -Сообщения Cap'n Proto строго типизированы и не самоописывающиеся, т.е. нуждаются во внешнем описании схемы. Схема применяется "на лету" и кешируется для каждого запроса. - -```sql -SELECT SearchPhrase, count() AS c FROM test.hits - GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' -``` - -Где `schema.capnp` выглядит следующим образом: - -``` -struct Message { - SearchPhrase @0 :Text; - c @1 :Uint64; -} -``` - - -Файлы со схемами находятся в файле, который находится в каталоге указанном в параметре [format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) конфигурации сервера. - -Десериализация эффективна и обычно не повышает нагрузку на систему. + + +# CapnProto + +Cap'n Proto - формат бинарных сообщений, похож на Protocol Buffers и Thrift, но не похож на JSON или MessagePack. + +Сообщения Cap'n Proto строго типизированы и не самоописывающиеся, т.е. нуждаются во внешнем описании схемы. Схема применяется "на лету" и кешируется для каждого запроса. + +```sql +SELECT SearchPhrase, count() AS c FROM test.hits + GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' +``` + +Где `schema.capnp` выглядит следующим образом: + +``` +struct Message { + SearchPhrase @0 :Text; + c @1 :Uint64; +} +``` + + +Файлы со схемами находятся в файле, который находится в каталоге указанном в параметре [format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) конфигурации сервера. + +Десериализация эффективна и обычно не повышает нагрузку на систему. diff --git a/docs/ru/operations/access_rights.md b/docs/ru/operations/access_rights.md index 7c0166219f5..54809b27ffe 100644 --- a/docs/ru/operations/access_rights.md +++ b/docs/ru/operations/access_rights.md @@ -61,7 +61,7 @@ Здесь видно объявление двух пользователей - `default` и `web`. Пользователя `web` мы добавили самостоятельно. -Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../table_engines/distributed.html)). +Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../table_engines/distributed.md#table_engines-distributed)). Пользователь, который используется для обмена информацией между серверами, объединенными в кластер, не должен иметь существенных ограничений или квот - иначе распределённые запросы сломаются. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index 513c003a735..67ba6ddbb29 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -67,7 +67,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat База данных по умолчанию. -Перечень баз данных можно получить запросом [SHOW DATABASES](../query_language/queries.md#query_language_queries_show_databases). +Перечень баз данных можно получить запросом [SHOW DATABASES](../../query_language/queries.md#query_language_queries_show_databases). **Пример** @@ -593,22 +593,6 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat Значение атрибута `incl` смотрите в разделе "[Конфигурационные файлы](../configuration_files.md#configuration_files)". - - -## resharding - -Путь в ZooKeeper к очереди задач. - -Подробнее читайте в разделе "[Перешардирование](../../table_engines/resharding.md#table_engines-resharding)". - -**Пример** - -```xml - - /clickhouse/task_queue - -``` - ## timezone diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 72b896d1505..af6989be3f8 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -101,7 +101,7 @@ ClickHouse применяет настройку в том случае, ког Установка логгирования запроса. -Запросы, переданные в ClickHouse с этой установкой, логгируются согласно правилам конфигурационного параметра сервера [query_log](../server_setings/settings.md#server_settings-query_log). +Запросы, переданные в ClickHouse с этой установкой, логгируются согласно правилам конфигурационного параметра сервера [query_log](../server_settings/settings.md#server_settings-query_log). **Пример** : diff --git a/docs/ru/table_engines/kafka.md b/docs/ru/table_engines/kafka.md index b3912e870fe..aa8ab29b3f0 100644 --- a/docs/ru/table_engines/kafka.md +++ b/docs/ru/table_engines/kafka.md @@ -1,77 +1,77 @@ -# Kafka - -Движок работает с [Apache Kafka](http://kafka.apache.org/). - -Kafka позволяет: - -- Публиковать/подписываться на потоки данных. -- Организовать отказо-устойчивое хранилище. -- Обрабатывать потоки по мере их появления. - -``` -Kafka(broker_list, topic_list, group_name, format[, schema]) -``` - -Параметры: -- `broker_list` - Перечень брокеров, разделенный запятыми (`localhost:9092`). -- `topic_list` - Перечень необходимых топиков Kafka (`my_topic`). -- `group_name` - Группа потребителя Kafka (`group1`). Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. -- `format` - Формат сообщений. Имеет те же обозначения, что выдает SQL-выражение `FORMAT`, например, `JSONEachRow`. -- `schema` - Опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - -Пример: - -```sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - SELECT * FROM queue LIMIT 5; -``` - -Полученные сообщения отслеживаются автоматически, поэтому из одной группы каждое сообщение считывается только один раз. Если необходимо получить данные дважды, то создайте копию таблицы с другим именем группы. - -Группы пластичны и синхронизированы на кластере. Например, если есть 10 топиков и 5 копий таблицы в кластере, то в каждую копию попадет по 2 топика. Если количество копий изменится, то распределение топиков по копиям изменится автоматически. Подробно читайте об этом на [http://kafka.apache.org/intro](http://kafka.apache.org/intro). - -Чтение сообщения с помощью `SELECT` не слишком полезно (разве что для отладки), поскольку каждое сообщения может быть прочитано только один раз. Практичнее создавать потоки реального времени с помощью материализованных преставлений. Для этого: - -1. Создайте потребителя Kafka с помощью движка и рассматривайте его как поток данных. -2. Создайте таблицу с необходимой структурой. -3. Создайте материализованное представление, которое преобразует данные от движка и помещает их в ранее созданную таблицу. - -Когда к движку присоединяется материализованное представление (`MATERIALIZED VIEW`), оно начинает в фоновом режиме собирать данные. Это позволяет непрерывно получать сообщения от Kafka и преобразовывать их в необходимый формат с помощью `SELECT`. - -Пример: - -```sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - CREATE TABLE daily ( - day Date, - level String, - total UInt64 - ) ENGINE = SummingMergeTree(day, (day, level), 8192); - - CREATE MATERIALIZED VIEW consumer TO daily - AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total - FROM queue GROUP BY day, level; - -SELECT level, sum(total) FROM daily GROUP BY level; -``` - -Для улучшения производительности полученные сообщения группируются в блоки размера [max_insert_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). Если блок не удалось сформировать за [stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) миллисекунд, то данные будут сброшены в таблицу независимо от полноты блока. - -Чтобы остановить получение данных топика или изменить логику преобразования, отсоедините материализованное представление: - -``` -DETACH TABLE consumer; -ATTACH MATERIALIZED VIEW consumer; -``` - +# Kafka + +Движок работает с [Apache Kafka](http://kafka.apache.org/). + +Kafka позволяет: + +- Публиковать/подписываться на потоки данных. +- Организовать отказо-устойчивое хранилище. +- Обрабатывать потоки по мере их появления. + +``` +Kafka(broker_list, topic_list, group_name, format[, schema]) +``` + +Параметры: +- `broker_list` - Перечень брокеров, разделенный запятыми (`localhost:9092`). +- `topic_list` - Перечень необходимых топиков Kafka (`my_topic`). +- `group_name` - Группа потребителя Kafka (`group1`). Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. +- `format` - Формат сообщений. Имеет те же обозначения, что выдает SQL-выражение `FORMAT`, например, `JSONEachRow`. +- `schema` - Опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. + +Пример: + +```sql + CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + SELECT * FROM queue LIMIT 5; +``` + +Полученные сообщения отслеживаются автоматически, поэтому из одной группы каждое сообщение считывается только один раз. Если необходимо получить данные дважды, то создайте копию таблицы с другим именем группы. + +Группы пластичны и синхронизированы на кластере. Например, если есть 10 топиков и 5 копий таблицы в кластере, то в каждую копию попадет по 2 топика. Если количество копий изменится, то распределение топиков по копиям изменится автоматически. Подробно читайте об этом на [http://kafka.apache.org/intro](http://kafka.apache.org/intro). + +Чтение сообщения с помощью `SELECT` не слишком полезно (разве что для отладки), поскольку каждое сообщения может быть прочитано только один раз. Практичнее создавать потоки реального времени с помощью материализованных преставлений. Для этого: + +1. Создайте потребителя Kafka с помощью движка и рассматривайте его как поток данных. +2. Создайте таблицу с необходимой структурой. +3. Создайте материализованное представление, которое преобразует данные от движка и помещает их в ранее созданную таблицу. + +Когда к движку присоединяется материализованное представление (`MATERIALIZED VIEW`), оно начинает в фоновом режиме собирать данные. Это позволяет непрерывно получать сообщения от Kafka и преобразовывать их в необходимый формат с помощью `SELECT`. + +Пример: + +```sql + CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + CREATE TABLE daily ( + day Date, + level String, + total UInt64 + ) ENGINE = SummingMergeTree(day, (day, level), 8192); + + CREATE MATERIALIZED VIEW consumer TO daily + AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total + FROM queue GROUP BY day, level; + +SELECT level, sum(total) FROM daily GROUP BY level; +``` + +Для улучшения производительности полученные сообщения группируются в блоки размера [max_insert_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). Если блок не удалось сформировать за [stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) миллисекунд, то данные будут сброшены в таблицу независимо от полноты блока. + +Чтобы остановить получение данных топика или изменить логику преобразования, отсоедините материализованное представление: + +``` +DETACH TABLE consumer; +ATTACH MATERIALIZED VIEW consumer; +``` + Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления. \ No newline at end of file From 13d9a4eebee35882f580e06573f1228064265312 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Dec 2017 18:13:23 +0300 Subject: [PATCH 3/3] Sources for english documentation switched to Markdown. Edit page link is fixed too for both language versions of documentation. --- docs/_static/custom.js | 6 +- docs/en/agg_functions/combinators.md | 40 + docs/en/agg_functions/index.md | 21 + docs/en/agg_functions/index.rst | 331 ---- docs/en/agg_functions/parametric_functions.md | 72 + docs/en/agg_functions/reference.md | 335 ++++ docs/en/conf.py | 11 +- docs/en/data_types/array.md | 5 + docs/en/data_types/array.rst | 6 - docs/en/data_types/boolean.md | 4 + docs/en/data_types/boolean.rst | 4 - docs/en/data_types/date.md | 7 + docs/en/data_types/date.rst | 7 - .../data_types/{datetime.rst => datetime.md} | 13 +- docs/en/data_types/enum.md | 31 + docs/en/data_types/enum.rst | 30 - docs/en/data_types/fixedstring.md | 10 + docs/en/data_types/fixedstring.rst | 10 - docs/en/data_types/float.md | 71 + docs/en/data_types/float.rst | 7 - docs/en/data_types/index.md | 13 + docs/en/data_types/int_uint.md | 18 + docs/en/data_types/int_uint.rst | 40 - ...egatefunction.rst => aggregatefunction.md} | 4 +- .../nested_data_structures/index.md | 9 + .../nested_data_structures/index.rst | 7 - .../nested_data_structures/nested.md | 98 ++ .../nested_data_structures/nested.rst | 99 -- .../{expression.rst => expression.md} | 4 +- .../{index.rst => index.md} | 6 +- .../special_data_types/{set.rst => set.md} | 4 +- docs/en/data_types/{string.rst => string.md} | 8 +- docs/en/data_types/{tuple.rst => tuple.md} | 6 +- docs/en/development/architecture.md | 195 +++ docs/en/development/architecture.rst | 227 --- docs/en/development/build.md | 128 ++ docs/en/development/build.rst | 147 -- docs/en/development/build_osx.md | 45 + docs/en/development/build_osx.rst | 63 - docs/en/development/index.md | 9 + docs/en/development/index.rst | 7 - docs/en/development/style.md | 813 +++++++++ docs/en/development/style.rst | 731 -------- docs/en/development/tests.md | 32 + docs/en/development/tests.rst | 38 - docs/en/dicts/external_dicts.md | 54 + docs/en/dicts/external_dicts.rst | 368 ---- docs/en/dicts/external_dicts_dict.md | 34 + docs/en/dicts/external_dicts_dict_layout.md | 222 +++ docs/en/dicts/external_dicts_dict_lifetime.md | 59 + docs/en/dicts/external_dicts_dict_sources.md | 397 +++++ .../en/dicts/external_dicts_dict_structure.md | 122 ++ docs/en/dicts/index.md | 15 + docs/en/dicts/index.rst | 11 - docs/en/dicts/internal_dicts.md | 49 + docs/en/dicts/internal_dicts.rst | 45 - docs/en/formats/capnproto.md | 26 + docs/en/formats/capnproto.rst | 29 - docs/en/formats/csv.md | 10 + docs/en/formats/csv.rst | 10 - docs/en/formats/csvwithnames.md | 4 + docs/en/formats/csvwithnames.rst | 4 - docs/en/formats/index.md | 13 + docs/en/formats/index.rst | 9 - docs/en/formats/json.md | 86 + docs/en/formats/json.rst | 87 - docs/en/formats/jsoncompact.md | 46 + docs/en/formats/jsoncompact.rst | 46 - docs/en/formats/jsoneachrow.md | 21 + docs/en/formats/jsoneachrow.rst | 25 - docs/en/formats/{native.rst => native.md} | 6 +- docs/en/formats/null.md | 5 + docs/en/formats/null.rst | 4 - docs/en/formats/pretty.md | 37 + docs/en/formats/pretty.rst | 36 - docs/en/formats/prettycompact.md | 5 + docs/en/formats/prettycompact.rst | 4 - docs/en/formats/prettycompactmonoblock.md | 4 + docs/en/formats/prettycompactmonoblock.rst | 4 - docs/en/formats/prettynoescapes.md | 20 + docs/en/formats/prettynoescapes.rst | 20 - docs/en/formats/prettyspace.md | 4 + docs/en/formats/prettyspace.rst | 4 - docs/en/formats/rowbinary.md | 13 + docs/en/formats/rowbinary.rst | 13 - docs/en/formats/tabseparated.md | 59 + docs/en/formats/tabseparated.rst | 59 - docs/en/formats/tabseparatedraw.md | 7 + docs/en/formats/tabseparatedraw.rst | 7 - docs/en/formats/tabseparatedwithnames.md | 8 + docs/en/formats/tabseparatedwithnames.rst | 8 - .../formats/tabseparatedwithnamesandtypes.md | 7 + .../formats/tabseparatedwithnamesandtypes.rst | 7 - docs/en/formats/tskv.md | 23 + docs/en/formats/tskv.rst | 21 - docs/en/formats/values.md | 9 + docs/en/formats/values.rst | 9 - docs/en/formats/{vertical.rst => vertical.md} | 6 +- docs/en/formats/xml.md | 73 + docs/en/formats/xml.rst | 74 - docs/en/functions/arithmetic_functions.md | 75 + docs/en/functions/arithmetic_functions.rst | 76 - docs/en/functions/array_functions.md | 323 ++++ docs/en/functions/array_functions.rst | 181 -- docs/en/functions/array_join.md | 31 + docs/en/functions/array_join.rst | 32 - .../{bit_functions.rst => bit_functions.md} | 22 +- docs/en/functions/comparison_functions.md | 31 + docs/en/functions/comparison_functions.rst | 36 - docs/en/functions/conditional_functions.md | 6 + docs/en/functions/conditional_functions.rst | 7 - ...e_functions.rst => date_time_functions.md} | 167 +- docs/en/functions/encoding_functions.md | 27 + docs/en/functions/encoding_functions.rst | 31 - docs/en/functions/ext_dict_functions.md | 46 + docs/en/functions/ext_dict_functions.rst | 46 - .../{hash_functions.rst => hash_functions.md} | 61 +- docs/en/functions/higher_order_functions.md | 73 + docs/en/functions/higher_order_functions.rst | 73 - docs/en/functions/in_functions.md | 18 + docs/en/functions/in_functions.rst | 18 - docs/en/functions/{index.rst => index.md} | 53 +- docs/en/functions/ip_address_functions.md | 115 ++ docs/en/functions/ip_address_functions.rst | 105 -- docs/en/functions/json_functions.md | 54 + docs/en/functions/json_functions.rst | 55 - docs/en/functions/logical_functions.md | 14 + docs/en/functions/logical_functions.rst | 19 - .../{math_functions.rst => math_functions.md} | 108 +- docs/en/functions/other_functions.md | 281 +++ docs/en/functions/other_functions.rst | 281 --- ...ndom_functions.rst => random_functions.md} | 13 +- ...ng_functions.rst => rounding_functions.md} | 46 +- .../functions/splitting_merging_functions.md | 20 + .../functions/splitting_merging_functions.rst | 23 - ...ring_functions.rst => string_functions.md} | 84 +- docs/en/functions/string_replace_functions.md | 79 + .../en/functions/string_replace_functions.rst | 80 - docs/en/functions/string_search_functions.md | 52 + docs/en/functions/string_search_functions.rst | 52 - .../en/functions/type_conversion_functions.md | 121 ++ .../functions/type_conversion_functions.rst | 132 -- docs/en/functions/url_functions.md | 121 ++ docs/en/functions/url_functions.rst | 123 -- docs/en/functions/ym_dict_functions.md | 119 ++ docs/en/functions/ym_dict_functions.rst | 125 -- .../example_datasets/amplab_benchmark.md | 121 ++ .../example_datasets/amplab_benchmark.rst | 124 -- .../example_datasets/criteo.md | 71 + .../example_datasets/criteo.rst | 73 - .../example_datasets/nyc_taxi.md | 371 ++++ .../example_datasets/nyc_taxi.rst | 387 ----- .../example_datasets/ontime.md | 319 ++++ .../example_datasets/ontime.rst | 318 ---- .../example_datasets/star_schema.md | 84 + .../example_datasets/star_schema.rst | 86 - .../example_datasets/wikistat.md | 27 + .../example_datasets/wikistat.rst | 27 - docs/en/getting_started/index.md | 143 ++ docs/en/getting_started/index.rst | 147 -- docs/en/{index.rst => index.md} | 6 +- docs/en/interfaces/cli.md | 112 ++ docs/en/interfaces/cli.rst | 110 -- .../{http_interface.rst => http_interface.md} | 255 +-- docs/en/interfaces/{index.rst => index.md} | 8 +- docs/en/interfaces/jdbc.md | 4 + docs/en/interfaces/jdbc.rst | 4 - docs/en/interfaces/{tcp.rst => tcp.md} | 4 +- .../third-party_client_libraries.md | 38 + .../third-party_client_libraries.rst | 37 - docs/en/interfaces/third-party_gui.md | 16 + docs/en/interfaces/third-party_gui.rst | 15 - ...e_features.rst => distinctive_features.md} | 74 +- .../features_considered_disadvantages.md | 6 + .../features_considered_disadvantages.rst | 6 - docs/en/introduction/{index.rst => index.md} | 7 +- .../{performance.rst => performance.md} | 27 +- .../introduction/possible_silly_questions.md | 15 + .../introduction/possible_silly_questions.rst | 19 - docs/en/introduction/what_is_clickhouse.md | 123 ++ docs/en/introduction/what_is_clickhouse.rst | 122 -- docs/en/introduction/ya_metrika_task.md | 47 + docs/en/introduction/ya_metrika_task.rst | 42 - docs/en/operations/access_rights.md | 98 ++ docs/en/operations/access_rights.rst | 71 - docs/en/operations/configuration_files.md | 28 + docs/en/operations/configuration_files.rst | 23 - .../index.rst => operations/index.md} | 6 +- docs/en/operations/index.rst | 8 - docs/en/operations/quotas.md | 101 ++ docs/en/operations/quotas.rst | 95 -- docs/en/operations/server_settings/index.md | 19 + .../en/operations/server_settings/settings.md | 705 ++++++++ docs/en/operations/settings/index.md | 31 + docs/en/operations/settings/index.rst | 29 - ...ery_complexity.rst => query_complexity.md} | 194 +-- docs/en/operations/settings/settings.md | 352 ++++ docs/en/operations/settings/settings.rst | 225 --- .../operations/settings/settings_profiles.md | 60 + .../operations/settings/settings_profiles.rst | 53 - docs/en/operations/tips.md | 252 +++ docs/en/operations/tips.rst | 280 --- docs/en/operators/index.md | 122 ++ docs/en/operators/index.rst | 138 -- docs/en/query_language/clickhouse_local.md | 4 + docs/en/query_language/clickhouse_local.rst | 4 - docs/en/query_language/index.md | 9 + docs/en/query_language/index.rst | 7 - docs/en/query_language/queries.md | 1508 +++++++++++++++++ docs/en/query_language/queries.rst | 1476 ---------------- docs/en/query_language/syntax.md | 106 ++ docs/en/query_language/syntax.rst | 102 -- docs/en/roadmap.md | 19 + docs/en/roadmap.rst | 20 - docs/en/system_tables/{index.rst => index.md} | 6 +- .../system.asynchronous_metrics.md | 8 + .../system.asynchronous_metrics.rst | 5 - docs/en/system_tables/system.clusters.md | 16 + docs/en/system_tables/system.clusters.rst | 16 - docs/en/system_tables/system.columns.md | 14 + docs/en/system_tables/system.columns.rst | 15 - docs/en/system_tables/system.databases.md | 6 + docs/en/system_tables/system.databases.rst | 6 - docs/en/system_tables/system.dictionaries.md | 24 + docs/en/system_tables/system.dictionaries.rst | 24 - .../{system.events.rst => system.events.md} | 8 +- docs/en/system_tables/system.functions.md | 11 + docs/en/system_tables/system.functions.rst | 10 - docs/en/system_tables/system.merges.md | 21 + docs/en/system_tables/system.merges.rst | 20 - docs/en/system_tables/system.metrics.md | 4 + docs/en/system_tables/system.metrics.rst | 2 - .../{system.numbers.rst => system.numbers.md} | 4 +- ...em.numbers_mt.rst => system.numbers_mt.md} | 4 +- .../{system.one.rst => system.one.md} | 4 +- docs/en/system_tables/system.parts.md | 29 + docs/en/system_tables/system.parts.rst | 20 - docs/en/system_tables/system.processes.md | 25 + docs/en/system_tables/system.processes.rst | 25 - docs/en/system_tables/system.replicas.md | 117 ++ docs/en/system_tables/system.replicas.rst | 116 -- docs/en/system_tables/system.settings.md | 30 + docs/en/system_tables/system.settings.rst | 29 - .../{system.tables.rst => system.tables.md} | 6 +- docs/en/system_tables/system.zookeeper.md | 73 + docs/en/system_tables/system.zookeeper.rst | 71 - docs/en/table_engines/aggregatingmergetree.md | 81 + .../en/table_engines/aggregatingmergetree.rst | 82 - .../table_engines/{buffer.rst => buffer.md} | 28 +- ...ngmergetree.rst => collapsingmergetree.md} | 23 +- .../table_engines/custom_partitioning_key.md | 47 + .../{distributed.rst => distributed.md} | 116 +- docs/en/table_engines/external_data.md | 60 + docs/en/table_engines/external_data.rst | 62 - docs/en/table_engines/file.md | 4 + docs/en/table_engines/file.rst | 4 - docs/en/table_engines/graphitemergetree.md | 86 + docs/en/table_engines/index.md | 21 + docs/en/table_engines/index.rst | 18 - docs/en/table_engines/{join.rst => join.md} | 13 +- docs/en/table_engines/kafka.md | 79 + docs/en/table_engines/kafka.rst | 111 -- docs/en/table_engines/{log.rst => log.md} | 7 +- docs/en/table_engines/materializedview.md | 4 + docs/en/table_engines/materializedview.rst | 4 - .../table_engines/{memory.rst => memory.md} | 4 +- docs/en/table_engines/{merge.rst => merge.md} | 35 +- .../{mergetree.rst => mergetree.md} | 44 +- docs/en/table_engines/{null.rst => null.md} | 6 +- docs/en/table_engines/replacingmergetree.md | 18 + docs/en/table_engines/replacingmergetree.rst | 19 - .../{replication.rst => replication.md} | 153 +- docs/en/table_engines/{set.rst => set.md} | 4 +- docs/en/table_engines/summingmergetree.md | 43 + docs/en/table_engines/summingmergetree.rst | 44 - .../table_engines/{tinylog.rst => tinylog.md} | 11 +- docs/en/table_engines/view.md | 4 + docs/en/table_engines/view.rst | 4 - .../table_functions/{index.rst => index.md} | 6 +- docs/en/table_functions/merge.md | 6 + docs/en/table_functions/merge.rst | 6 - docs/en/table_functions/remote.md | 78 + docs/en/table_functions/remote.rst | 73 - docs/ru/table_engines/dictionary.md | 19 +- 284 files changed, 11680 insertions(+), 9831 deletions(-) create mode 100644 docs/en/agg_functions/combinators.md create mode 100644 docs/en/agg_functions/index.md delete mode 100644 docs/en/agg_functions/index.rst create mode 100644 docs/en/agg_functions/parametric_functions.md create mode 100644 docs/en/agg_functions/reference.md create mode 100644 docs/en/data_types/array.md delete mode 100644 docs/en/data_types/array.rst create mode 100644 docs/en/data_types/boolean.md delete mode 100644 docs/en/data_types/boolean.rst create mode 100644 docs/en/data_types/date.md delete mode 100644 docs/en/data_types/date.rst rename docs/en/data_types/{datetime.rst => datetime.md} (68%) create mode 100644 docs/en/data_types/enum.md delete mode 100644 docs/en/data_types/enum.rst create mode 100644 docs/en/data_types/fixedstring.md delete mode 100644 docs/en/data_types/fixedstring.rst create mode 100644 docs/en/data_types/float.md delete mode 100644 docs/en/data_types/float.rst create mode 100644 docs/en/data_types/index.md create mode 100644 docs/en/data_types/int_uint.md delete mode 100644 docs/en/data_types/int_uint.rst rename docs/en/data_types/nested_data_structures/{aggregatefunction.rst => aggregatefunction.md} (63%) create mode 100644 docs/en/data_types/nested_data_structures/index.md delete mode 100644 docs/en/data_types/nested_data_structures/index.rst create mode 100644 docs/en/data_types/nested_data_structures/nested.md delete mode 100644 docs/en/data_types/nested_data_structures/nested.rst rename docs/en/data_types/special_data_types/{expression.rst => expression.md} (75%) rename docs/en/data_types/special_data_types/{index.rst => index.md} (81%) rename docs/en/data_types/special_data_types/{set.rst => set.md} (85%) rename docs/en/data_types/{string.rst => string.md} (96%) rename docs/en/data_types/{tuple.rst => tuple.md} (66%) create mode 100644 docs/en/development/architecture.md delete mode 100644 docs/en/development/architecture.rst create mode 100644 docs/en/development/build.md delete mode 100644 docs/en/development/build.rst create mode 100644 docs/en/development/build_osx.md delete mode 100644 docs/en/development/build_osx.rst create mode 100644 docs/en/development/index.md delete mode 100644 docs/en/development/index.rst create mode 100644 docs/en/development/style.md delete mode 100644 docs/en/development/style.rst create mode 100644 docs/en/development/tests.md delete mode 100644 docs/en/development/tests.rst create mode 100644 docs/en/dicts/external_dicts.md delete mode 100644 docs/en/dicts/external_dicts.rst create mode 100644 docs/en/dicts/external_dicts_dict.md create mode 100644 docs/en/dicts/external_dicts_dict_layout.md create mode 100644 docs/en/dicts/external_dicts_dict_lifetime.md create mode 100644 docs/en/dicts/external_dicts_dict_sources.md create mode 100644 docs/en/dicts/external_dicts_dict_structure.md create mode 100644 docs/en/dicts/index.md delete mode 100644 docs/en/dicts/index.rst create mode 100644 docs/en/dicts/internal_dicts.md delete mode 100644 docs/en/dicts/internal_dicts.rst create mode 100644 docs/en/formats/capnproto.md delete mode 100644 docs/en/formats/capnproto.rst create mode 100644 docs/en/formats/csv.md delete mode 100644 docs/en/formats/csv.rst create mode 100644 docs/en/formats/csvwithnames.md delete mode 100644 docs/en/formats/csvwithnames.rst create mode 100644 docs/en/formats/index.md delete mode 100644 docs/en/formats/index.rst create mode 100644 docs/en/formats/json.md delete mode 100644 docs/en/formats/json.rst create mode 100644 docs/en/formats/jsoncompact.md delete mode 100644 docs/en/formats/jsoncompact.rst create mode 100644 docs/en/formats/jsoneachrow.md delete mode 100644 docs/en/formats/jsoneachrow.rst rename docs/en/formats/{native.rst => native.md} (67%) create mode 100644 docs/en/formats/null.md delete mode 100644 docs/en/formats/null.rst create mode 100644 docs/en/formats/pretty.md delete mode 100644 docs/en/formats/pretty.rst create mode 100644 docs/en/formats/prettycompact.md delete mode 100644 docs/en/formats/prettycompact.rst create mode 100644 docs/en/formats/prettycompactmonoblock.md delete mode 100644 docs/en/formats/prettycompactmonoblock.rst create mode 100644 docs/en/formats/prettynoescapes.md delete mode 100644 docs/en/formats/prettynoescapes.rst create mode 100644 docs/en/formats/prettyspace.md delete mode 100644 docs/en/formats/prettyspace.rst create mode 100644 docs/en/formats/rowbinary.md delete mode 100644 docs/en/formats/rowbinary.rst create mode 100644 docs/en/formats/tabseparated.md delete mode 100644 docs/en/formats/tabseparated.rst create mode 100644 docs/en/formats/tabseparatedraw.md delete mode 100644 docs/en/formats/tabseparatedraw.rst create mode 100644 docs/en/formats/tabseparatedwithnames.md delete mode 100644 docs/en/formats/tabseparatedwithnames.rst create mode 100644 docs/en/formats/tabseparatedwithnamesandtypes.md delete mode 100644 docs/en/formats/tabseparatedwithnamesandtypes.rst create mode 100644 docs/en/formats/tskv.md delete mode 100644 docs/en/formats/tskv.rst create mode 100644 docs/en/formats/values.md delete mode 100644 docs/en/formats/values.rst rename docs/en/formats/{vertical.rst => vertical.md} (57%) create mode 100644 docs/en/formats/xml.md delete mode 100644 docs/en/formats/xml.rst create mode 100644 docs/en/functions/arithmetic_functions.md delete mode 100644 docs/en/functions/arithmetic_functions.rst create mode 100644 docs/en/functions/array_functions.md delete mode 100644 docs/en/functions/array_functions.rst create mode 100644 docs/en/functions/array_join.md delete mode 100644 docs/en/functions/array_join.rst rename docs/en/functions/{bit_functions.rst => bit_functions.md} (63%) create mode 100644 docs/en/functions/comparison_functions.md delete mode 100644 docs/en/functions/comparison_functions.rst create mode 100644 docs/en/functions/conditional_functions.md delete mode 100644 docs/en/functions/conditional_functions.rst rename docs/en/functions/{date_time_functions.rst => date_time_functions.md} (53%) create mode 100644 docs/en/functions/encoding_functions.md delete mode 100644 docs/en/functions/encoding_functions.rst create mode 100644 docs/en/functions/ext_dict_functions.md delete mode 100644 docs/en/functions/ext_dict_functions.rst rename docs/en/functions/{hash_functions.rst => hash_functions.md} (76%) create mode 100644 docs/en/functions/higher_order_functions.md delete mode 100644 docs/en/functions/higher_order_functions.rst create mode 100644 docs/en/functions/in_functions.md delete mode 100644 docs/en/functions/in_functions.rst rename docs/en/functions/{index.rst => index.md} (71%) create mode 100644 docs/en/functions/ip_address_functions.md delete mode 100644 docs/en/functions/ip_address_functions.rst create mode 100644 docs/en/functions/json_functions.md delete mode 100644 docs/en/functions/json_functions.rst create mode 100644 docs/en/functions/logical_functions.md delete mode 100644 docs/en/functions/logical_functions.rst rename docs/en/functions/{math_functions.rst => math_functions.md} (62%) create mode 100644 docs/en/functions/other_functions.md delete mode 100644 docs/en/functions/other_functions.rst rename docs/en/functions/{random_functions.rst => random_functions.md} (84%) rename docs/en/functions/{rounding_functions.rst => rounding_functions.md} (51%) create mode 100644 docs/en/functions/splitting_merging_functions.md delete mode 100644 docs/en/functions/splitting_merging_functions.rst rename docs/en/functions/{string_functions.rst => string_functions.md} (53%) create mode 100644 docs/en/functions/string_replace_functions.md delete mode 100644 docs/en/functions/string_replace_functions.rst create mode 100644 docs/en/functions/string_search_functions.md delete mode 100644 docs/en/functions/string_search_functions.rst create mode 100644 docs/en/functions/type_conversion_functions.md delete mode 100644 docs/en/functions/type_conversion_functions.rst create mode 100644 docs/en/functions/url_functions.md delete mode 100644 docs/en/functions/url_functions.rst create mode 100644 docs/en/functions/ym_dict_functions.md delete mode 100644 docs/en/functions/ym_dict_functions.rst create mode 100644 docs/en/getting_started/example_datasets/amplab_benchmark.md delete mode 100644 docs/en/getting_started/example_datasets/amplab_benchmark.rst create mode 100644 docs/en/getting_started/example_datasets/criteo.md delete mode 100644 docs/en/getting_started/example_datasets/criteo.rst create mode 100644 docs/en/getting_started/example_datasets/nyc_taxi.md delete mode 100644 docs/en/getting_started/example_datasets/nyc_taxi.rst create mode 100644 docs/en/getting_started/example_datasets/ontime.md delete mode 100644 docs/en/getting_started/example_datasets/ontime.rst create mode 100644 docs/en/getting_started/example_datasets/star_schema.md delete mode 100644 docs/en/getting_started/example_datasets/star_schema.rst create mode 100644 docs/en/getting_started/example_datasets/wikistat.md delete mode 100644 docs/en/getting_started/example_datasets/wikistat.rst create mode 100644 docs/en/getting_started/index.md delete mode 100644 docs/en/getting_started/index.rst rename docs/en/{index.rst => index.md} (91%) create mode 100644 docs/en/interfaces/cli.md delete mode 100644 docs/en/interfaces/cli.rst rename docs/en/interfaces/{http_interface.rst => http_interface.md} (50%) rename docs/en/interfaces/{index.rst => index.md} (73%) create mode 100644 docs/en/interfaces/jdbc.md delete mode 100644 docs/en/interfaces/jdbc.rst rename docs/en/interfaces/{tcp.rst => tcp.md} (82%) create mode 100644 docs/en/interfaces/third-party_client_libraries.md delete mode 100644 docs/en/interfaces/third-party_client_libraries.rst create mode 100644 docs/en/interfaces/third-party_gui.md delete mode 100644 docs/en/interfaces/third-party_gui.rst rename docs/en/introduction/{distinctive_features.rst => distinctive_features.md} (58%) create mode 100644 docs/en/introduction/features_considered_disadvantages.md delete mode 100644 docs/en/introduction/features_considered_disadvantages.rst rename docs/en/introduction/{index.rst => index.md} (81%) rename docs/en/introduction/{performance.rst => performance.md} (76%) create mode 100644 docs/en/introduction/possible_silly_questions.md delete mode 100644 docs/en/introduction/possible_silly_questions.rst create mode 100644 docs/en/introduction/what_is_clickhouse.md delete mode 100644 docs/en/introduction/what_is_clickhouse.rst create mode 100644 docs/en/introduction/ya_metrika_task.md delete mode 100644 docs/en/introduction/ya_metrika_task.rst create mode 100644 docs/en/operations/access_rights.md delete mode 100644 docs/en/operations/access_rights.rst create mode 100644 docs/en/operations/configuration_files.md delete mode 100644 docs/en/operations/configuration_files.rst rename docs/en/{data_types/index.rst => operations/index.md} (60%) delete mode 100644 docs/en/operations/index.rst create mode 100644 docs/en/operations/quotas.md delete mode 100644 docs/en/operations/quotas.rst create mode 100644 docs/en/operations/server_settings/index.md create mode 100644 docs/en/operations/server_settings/settings.md create mode 100644 docs/en/operations/settings/index.md delete mode 100644 docs/en/operations/settings/index.rst rename docs/en/operations/settings/{query_complexity.rst => query_complexity.md} (54%) create mode 100644 docs/en/operations/settings/settings.md delete mode 100644 docs/en/operations/settings/settings.rst create mode 100644 docs/en/operations/settings/settings_profiles.md delete mode 100644 docs/en/operations/settings/settings_profiles.rst create mode 100644 docs/en/operations/tips.md delete mode 100644 docs/en/operations/tips.rst create mode 100644 docs/en/operators/index.md delete mode 100644 docs/en/operators/index.rst create mode 100644 docs/en/query_language/clickhouse_local.md delete mode 100644 docs/en/query_language/clickhouse_local.rst create mode 100644 docs/en/query_language/index.md delete mode 100644 docs/en/query_language/index.rst create mode 100644 docs/en/query_language/queries.md delete mode 100644 docs/en/query_language/queries.rst create mode 100644 docs/en/query_language/syntax.md delete mode 100644 docs/en/query_language/syntax.rst create mode 100644 docs/en/roadmap.md delete mode 100644 docs/en/roadmap.rst rename docs/en/system_tables/{index.rst => index.md} (93%) create mode 100644 docs/en/system_tables/system.asynchronous_metrics.md delete mode 100644 docs/en/system_tables/system.asynchronous_metrics.rst create mode 100644 docs/en/system_tables/system.clusters.md delete mode 100644 docs/en/system_tables/system.clusters.rst create mode 100644 docs/en/system_tables/system.columns.md delete mode 100644 docs/en/system_tables/system.columns.rst create mode 100644 docs/en/system_tables/system.databases.md delete mode 100644 docs/en/system_tables/system.databases.rst create mode 100644 docs/en/system_tables/system.dictionaries.md delete mode 100644 docs/en/system_tables/system.dictionaries.rst rename docs/en/system_tables/{system.events.rst => system.events.md} (56%) create mode 100644 docs/en/system_tables/system.functions.md delete mode 100644 docs/en/system_tables/system.functions.rst create mode 100644 docs/en/system_tables/system.merges.md delete mode 100644 docs/en/system_tables/system.merges.rst create mode 100644 docs/en/system_tables/system.metrics.md delete mode 100644 docs/en/system_tables/system.metrics.rst rename docs/en/system_tables/{system.numbers.rst => system.numbers.md} (89%) rename docs/en/system_tables/{system.numbers_mt.rst => system.numbers_mt.md} (76%) rename docs/en/system_tables/{system.one.rst => system.one.md} (90%) create mode 100644 docs/en/system_tables/system.parts.md delete mode 100644 docs/en/system_tables/system.parts.rst create mode 100644 docs/en/system_tables/system.processes.md delete mode 100644 docs/en/system_tables/system.processes.rst create mode 100644 docs/en/system_tables/system.replicas.md delete mode 100644 docs/en/system_tables/system.replicas.rst create mode 100644 docs/en/system_tables/system.settings.md delete mode 100644 docs/en/system_tables/system.settings.rst rename docs/en/system_tables/{system.tables.rst => system.tables.md} (76%) create mode 100644 docs/en/system_tables/system.zookeeper.md delete mode 100644 docs/en/system_tables/system.zookeeper.rst create mode 100644 docs/en/table_engines/aggregatingmergetree.md delete mode 100644 docs/en/table_engines/aggregatingmergetree.rst rename docs/en/table_engines/{buffer.rst => buffer.md} (80%) rename docs/en/table_engines/{collapsingmergetree.rst => collapsingmergetree.md} (73%) create mode 100644 docs/en/table_engines/custom_partitioning_key.md rename docs/en/table_engines/{distributed.rst => distributed.md} (63%) create mode 100644 docs/en/table_engines/external_data.md delete mode 100644 docs/en/table_engines/external_data.rst create mode 100644 docs/en/table_engines/file.md delete mode 100644 docs/en/table_engines/file.rst create mode 100644 docs/en/table_engines/graphitemergetree.md create mode 100644 docs/en/table_engines/index.md delete mode 100644 docs/en/table_engines/index.rst rename docs/en/table_engines/{join.rst => join.md} (57%) create mode 100644 docs/en/table_engines/kafka.md delete mode 100644 docs/en/table_engines/kafka.rst rename docs/en/table_engines/{log.rst => log.md} (54%) create mode 100644 docs/en/table_engines/materializedview.md delete mode 100644 docs/en/table_engines/materializedview.rst rename docs/en/table_engines/{memory.rst => memory.md} (98%) rename docs/en/table_engines/{merge.rst => merge.md} (53%) rename docs/en/table_engines/{mergetree.rst => mergetree.md} (71%) rename docs/en/table_engines/{null.rst => null.md} (68%) create mode 100644 docs/en/table_engines/replacingmergetree.md delete mode 100644 docs/en/table_engines/replacingmergetree.rst rename docs/en/table_engines/{replication.rst => replication.md} (62%) rename docs/en/table_engines/{set.rst => set.md} (99%) create mode 100644 docs/en/table_engines/summingmergetree.md delete mode 100644 docs/en/table_engines/summingmergetree.rst rename docs/en/table_engines/{tinylog.rst => tinylog.md} (77%) create mode 100644 docs/en/table_engines/view.md delete mode 100644 docs/en/table_engines/view.rst rename docs/en/table_functions/{index.rst => index.md} (87%) create mode 100644 docs/en/table_functions/merge.md delete mode 100644 docs/en/table_functions/merge.rst create mode 100644 docs/en/table_functions/remote.md delete mode 100644 docs/en/table_functions/remote.rst diff --git a/docs/_static/custom.js b/docs/_static/custom.js index c706401de3d..1cd8b53f736 100644 --- a/docs/_static/custom.js +++ b/docs/_static/custom.js @@ -4,7 +4,7 @@ $(function() { var pathname = window.location.pathname; var url; if (pathname.indexOf('html') >= 0) { - url = pathname.replace('/docs/', 'https://github.com/yandex/ClickHouse/edit/master/docs/').replace('html', 'rst'); + url = pathname.replace('/docs/', 'https://github.com/yandex/ClickHouse/edit/master/docs/').replace('html', 'md'); } else { if (pathname.indexOf('/single/') >= 0) { if (pathname.indexOf('ru') >= 0) { @@ -14,9 +14,9 @@ $(function() { } } else { if (pathname.indexOf('ru') >= 0) { - url = 'https://github.com/yandex/ClickHouse/edit/master/docs/ru/index.rst'; + url = 'https://github.com/yandex/ClickHouse/edit/master/docs/ru/index.md'; } else { - url = 'https://github.com/yandex/ClickHouse/edit/master/docs/en/index.rst'; + url = 'https://github.com/yandex/ClickHouse/edit/master/docs/en/index.md'; } } } diff --git a/docs/en/agg_functions/combinators.md b/docs/en/agg_functions/combinators.md new file mode 100644 index 00000000000..ca5c4674172 --- /dev/null +++ b/docs/en/agg_functions/combinators.md @@ -0,0 +1,40 @@ + + +# Aggregate function combinators + +The name of an aggregate function can have a suffix appended to it. This changes the way the aggregate function works. + +## -If + +The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). + +Examples: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` and so on. + +With conditional aggregate functions, you can calculate aggregates for several conditions at once, without using subqueries and `JOIN`s. For example, in Yandex.Metrica, conditional aggregate functions are used to implement the segment comparison functionality. + +## -Array + +The -Array suffix can be appended to any aggregate function. In this case, the aggregate function takes arguments of the 'Array(T)' type (arrays) instead of 'T' type arguments. If the aggregate function accepts multiple arguments, this must be arrays of equal lengths. When processing arrays, the aggregate function works like the original aggregate function across all array elements. + +Example 1: `sumArray(arr)` - Totals all the elements of all 'arr' arrays. In this example, it could have been written more simply: `sum(arraySum(arr))`. + +Example 2: `uniqArray(arr)` – Count the number of unique elements in all 'arr' arrays. This could be done an easier way: `uniq(arrayJoin(arr))`, but it's not always possible to add 'arrayJoin' to a query. + +-If and -Array can be combined. However, 'Array' must come first, then 'If'. Examples: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. Due to this order, the 'cond' argument can't be an array. + +## -State + +If you apply this combinator, the aggregate function doesn't return the resulting value (such as the number of unique values for the 'uniq' function), but an intermediate state of the aggregation (for ` uniq`, this is the hash table for calculating the number of unique values). This is an AggregateFunction(...) that can be used for further processing or stored in a table to finish aggregating later. See the sections "AggregatingMergeTree" and "Functions for working with intermediate aggregation states". + +## -Merge + +If you apply this combinator, the aggregate function takes the intermediate aggregation state as an argument, combines the states to finish aggregation, and returns the resulting value. + +## -MergeState. + +Merges the intermediate aggregation states in the same way as the -Merge combinator. However, it doesn't return the resulting value, but an intermediate aggregation state, similar to the -State combinator. + +## -ForEach + +Converts an aggregate function for tables into an aggregate function for arrays that aggregates the corresponding array items and returns an array of results. For example, `sumForEach` for the arrays `[1, 2]`, `[3, 4, 5]`and`[6, 7]`returns the result `[10, 13, 5]` after adding together the corresponding array items. + diff --git a/docs/en/agg_functions/index.md b/docs/en/agg_functions/index.md new file mode 100644 index 00000000000..1ca9ca3283d --- /dev/null +++ b/docs/en/agg_functions/index.md @@ -0,0 +1,21 @@ + + +# Aggregate functions + +Aggregate functions work in the [normal](http://www.sql-tutorial.com/sql-aggregate-functions-sql-tutorial) way as expected by database experts. + +ClickHouse also supports: + +- [Parametric aggregate functions](parametric_functions.md#aggregate_functions_parametric), which accept other parameters in addition to columns. +- [Combinators](combinators.md#aggregate_functions_combinators), which change the behavior of aggregate functions. + +**Table of Contents** + +```eval_rst +.. toctree:: + + reference + parametric_functions + combinators +``` + diff --git a/docs/en/agg_functions/index.rst b/docs/en/agg_functions/index.rst deleted file mode 100644 index 8813345a268..00000000000 --- a/docs/en/agg_functions/index.rst +++ /dev/null @@ -1,331 +0,0 @@ -Aggregate functions -=================== - -count() -------- -Counts the number of rows. Accepts zero arguments and returns UInt64. -The syntax ``COUNT(DISTINCT x)`` is not supported. The separate ``uniq`` aggregate function exists for this purpose. - -A ``SELECT count() FROM table`` query is not optimized, because the number of entries in the table is not stored separately. It will select some small column from the table and count the number of values in it. - -any(x) ------- -Selects the first encountered value. -The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate. -To get a determinate result, you can use the ``min`` or ``max`` function instead of ``any``. - -In some cases, you can rely on the order of execution. This applies to cases when ``SELECT`` comes from a subquery that uses ``ORDER BY``. - -When a SELECT query has the GROUP BY clause or at least one aggregate function, ClickHouse (in contrast to for example MySQL) requires that all expressions in the ``SELECT``, ``HAVING`` and ``ORDER BY`` clauses be calculated from keys or from aggregate functions. That is, each column selected from the table must be used either in keys, or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the ``any`` aggregate function. - -anyLast(x) ----------- -Selects the last value encountered. -The result is just as indeterminate as for the 'any' function. - -min(x) ------- -Calculates the minimum. - -max(x) ------- -Calculates the maximum - -argMin(arg, val) ----------------- -Calculates the 'arg' value for a minimal 'val' value. If there are several different values of 'arg' for minimal values of 'val', the first of these values encountered is output. - -argMax(arg, val) ----------------- -Calculates the 'arg' value for a maximum 'val' value. If there are several different values of 'arg' for maximum values of 'val', the first of these values encountered is output. - -sum(x) ------- -Calculates the sum. -Only works for numbers. - -sumWithOverflow(x) ------------------- -Calculates the sum of numbers using the same data type as the input. If the sum of values is larger than the maximum value for given type, it will overflow. Only works for numbers. - -sumMap(key, value) ------- -Performs summation of array 'value' by corresponding keys of array 'key'. -Number of elements in 'key' and 'value' arrays should be the same for each row, on which summation is being performed. -Returns a tuple of two arrays - sorted keys and values, summed up by corresponding keys. - -Example: - -.. code-block:: sql - - CREATE TABLE sum_map( - date Date, - timeslot DateTime, - statusMap Nested( - status UInt16, - requests UInt64 - ) - ) ENGINE = Log; - INSERT INTO sum_map VALUES - ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), - ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); - SELECT - timeslot, - sumMap(statusMap.status, statusMap.requests) - FROM sum_map - GROUP BY timeslot - -.. code-block:: text - - ┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ - │ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ - │ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ - └─────────────────────┴──────────────────────────────────────────────┘ - -avg(x) ------- -Calculates the average. -Only works for numbers. -The result is always Float64. - -uniq(x) -------- -Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, and dates with times. - -Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65535. -Compared with the widely known `HyperLogLog `_ algorithm, this algorithm is less effective in terms of accuracy and memory consumption (even up to proportionality), but it is adaptive. This means that with fairly high accuracy, it consumes less memory during simultaneous computation of cardinality for a large number of data sets whose cardinality has power law distribution (i.e. in cases when most of the data sets are small). This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using ``uniq`` is almost as fast as using other aggregate functions). - -There is no compensation for the bias of an estimate, so for large data sets the results are systematically deflated. This function is normally used for computing the number of unique visitors in Yandex.Metrica, so this bias does not play a role. - -The result is deterministic (it does not depend on the order of query execution). - -uniqCombined(x) ---------------- -Approximately computes the number of different values ​​of the argument. Works for numbers, strings, dates, date-with-time, for several arguments and arguments-tuples. - -A combination of three algorithms is used: an array, a hash table and `HyperLogLog `_ with an error correction table. The memory consumption is several times smaller than the ``uniq`` function, and the accuracy is several times higher. The speed of operation is slightly lower than that of the ``uniq`` function, but sometimes it can be even higher - in the case of distributed requests, in which a large number of aggregation states are transmitted over the network. The maximum state size is 96 KiB (HyperLogLog of 217 6-bit cells). - -The result is deterministic (it does not depend on the order of query execution). - -The ``uniqCombined`` function is a good default choice for calculating the number of different values. - -uniqHLL12(x) ------------- -Uses the `HyperLogLog `_ algorithm to approximate the number of different values of the argument. It uses 212 5-bit cells. The size of the state is slightly more than 2.5 KB. - -The result is deterministic (it does not depend on the order of query execution). - -In most cases, use the 'uniq' function. You should only use this function if you understand its advantages well. - -uniqExact(x) ------------- -Calculates the number of different values of the argument, exactly. -There is no reason to fear approximations, so it's better to use the ``uniq`` function. -You should use the ``uniqExact`` function if you definitely need an exact result. - -The ``uniqExact`` function uses more memory than the ``uniq`` function, because the size of the state has unbounded growth as the number of different values increases. - -groupArray(x), groupArray(max_size)(x) --------------------------------------- -Creates an array of argument values. -Values can be added to the array in any (indeterminate) order. - -The second version (with ``max_size`` parameter) limits the size of resulting array to ``max_size`` elements. -For example, ``groupArray(1)(x)`` is equivalent to ``[any(x)]``. - -In some cases, you can rely on the order of execution. This applies to cases when ``SELECT`` comes from a subquery that uses ``ORDER BY``. - -groupUniqArray(x) ------------------ -Creates an array from different argument values. Memory consumption is the same as for the ``uniqExact`` function. - -quantile(level)(x) ------------------- -Approximates the 'level' quantile. 'level' is a constant, a floating-point number from 0 to 1. We recommend using a 'level' value in the range of 0.01..0.99. -Don't use a 'level' value equal to 0 or 1 - use the 'min' and 'max' functions for these cases. - -The algorithm is the same as for the ``median`` function. Actually, ``quantile`` and ``median`` are internally the same function. You can use the ``quantile`` function without parameters - in this case, it calculates the median, and you can use the ``median`` function with parameters - in this case, it calculates the quantile of the set level. - -When using multiple ``quantile` and ``median`` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the ``quantiles`` function. - -quantileDeterministic(level)(x, determinator) ---------------------------------------------- -Calculates the quantile of 'level' using the same algorithm as the ``medianDeterministic`` function. - - -quantileTiming(level)(x) ------------------------- -Calculates the quantile of 'level' using the same algorithm as the ``medianTiming`` function. - -quantileTimingWeighted(level)(x, weight) ----------------------------------------- -Calculates the quantile of 'level' using the same algorithm as the ``medianTimingWeighted`` function. - -quantileExact(level)(x) ------------------------ -Computes the level quantile exactly. To do this, all transferred values are added to an array, which is then partially sorted. Therefore, the function consumes O(n) memory, where n is the number of transferred values. However, for a small number of values, the function is very effective. - -quantileExactWeighted(level)(x, weight) ---------------------------------------- -Computes the level quantile exactly. In this case, each value is taken into account with the weight weight - as if it is present weight once. The arguments of the function can be considered as histograms, where the value "x" corresponds to the "column" of the histogram of the height weight, and the function itself can be considered as the summation of histograms. - -The algorithm is a hash table. Because of this, in case the transmitted values ​​are often repeated, the function consumes less RAM than the quantileExact. You can use this function instead of quantileExact, specifying the number 1 as the weight. - -quantileTDigest(level)(x) -------------------------- -Computes the level quantile approximately, using the `t-digest `_ algorithm. The maximum error is 1%. The memory consumption per state is proportional to the logarithm of the number of transmitted values. - -The performance of the function is below quantile, quantileTiming. By the ratio of state size and accuracy, the function is significantly better than quantile. - -The result depends on the order in which the query is executed, and is nondeterministic. - -median ------- -Approximates the median. Also see the similar ``quantile`` function. -Works for numbers, dates, and dates with times. -For numbers it returns Float64, for dates - a date, and for dates with times - a date with time. - -Uses `reservoir sampling `_ with a reservoir size up to 8192. -If necessary, the result is output with linear approximation from the two neighboring values. -This algorithm proved to be more practical than another well-known algorithm - QDigest. - -The result depends on the order of running the query, and is nondeterministic. - -quantiles(level1, level2, ...)(x) ---------------------------------- -Approximates quantiles of all specified levels. -The result is an array containing the corresponding number of values. - -varSamp(x) ----------- -Calculates the amount ``Σ((x - x̅)2) / (n - 1)``, where 'n' is the sample size and 'x̅' is the average value of 'x'. - -It represents an unbiased estimate of the variance of a random variable, if the values passed to the function are a sample of this random amount. - -Returns Float64. If n <= 1, it returns +∞. - -varPop(x) ---------- -Calculates the amount ``Σ((x - x̅)2) / n``, where 'n' is the sample size and 'x̅' is the average value of 'x'. - -In other words, dispersion for a set of values. Returns Float64. - -stddevSamp(x) -------------- -The result is equal to the square root of ``varSamp(x)``. - - -stddevPop(x) ------------- -The result is equal to the square root of ``varPop(x)``. - - -covarSamp(x, y) ---------------- -Calculates the value of ``Σ((x - x̅)(y - y̅)) / (n - 1)``. - -Returns Float64. If n <= 1, it returns +∞. - -covarPop(x, y) --------------- -Calculates the value of ``Σ((x - x̅)(y - y̅)) / n``. - -corr(x, y) ----------- -Calculates the Pearson correlation coefficient: ``Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)2) * Σ((y - y̅)2))``. - -Parametric aggregate functions -============================== -Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters - constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. - -sequenceMatch(pattern)(time, cond1, cond2, ...) ------------------------------------------------ -Pattern matching for event chains. - -'pattern' is a string containing a pattern to match. The pattern is similar to a regular expression. -'time' is the event time of the DateTime type. -'cond1, cond2 ...' are from one to 32 arguments of the UInt8 type that indicate whether an event condition was met. - -The function collects a sequence of events in RAM. Then it checks whether this sequence matches the pattern. -It returns UInt8 - 0 if the pattern isn't matched, or 1 if it matches. - -Example: ``sequenceMatch('(?1).*(?2)')(EventTime, URL LIKE '%company%', URL LIKE '%cart%')`` -- whether there was a chain of events in which pages with the address in company were visited earlier than pages with the address in cart. - -This is a simple example. You could write it using other aggregate functions: - -.. code-block:: sql - - minIf(EventTime, URL LIKE '%company%') < maxIf(EventTime, URL LIKE '%cart%'). - -However, there is no such solution for more complex situations. - -Pattern syntax: -``(?1)`` - Reference to a condition (any number in place of 1). -``.*`` - Any number of events. -``(?t>=1800)`` - Time condition. -Any quantity of any type of events is allowed over the specified time. -The operators <, >, <= may be used instead of >=. -Any number may be specified in place of 1800. - -Events that occur during the same second may be put in the chain in any order. This may affect the result of the function. - -sequenceCount(pattern)(time, cond1, cond2, ...) ------------------------------------------------ -Similar to the sequenceMatch function, but it does not return the fact that there is a chain of events, and UInt64 is the number of strings found. -Chains are searched without overlapping. That is, the following chain can start only after the end of the previous one. - -uniqUpTo(N)(x) --------------- -Calculates the number of different argument values, if it is less than or equal to N. -If the number of different argument values is greater than N, it returns N + 1. - -Recommended for use with small Ns, up to 10. The maximum N value is 100. - -For the state of an aggregate function, it uses the amount of memory equal to 1 + N * the size of one value of bytes. -For strings, it stores a non-cryptographic hash of 8 bytes. That is, the calculation is approximated for strings. - -It works as fast as possible, except for cases when a large N value is used and the number of unique values is slightly less than N. - -Usage example: -Problem: Generate a report that shows only keywords that produced at least 5 unique users. -Solution: Write in the query ``GROUP BY SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5`` - -Aggregate function combinators -============================== -The name of an aggregate function can have a suffix appended to it. This changes the way the aggregate function works. -There are ``If`` and ``Array`` combinators. See the sections below. - -If combinator. Conditional aggregate functions ----------------------------------------------- -The suffix ``-If`` can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument - a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). - -Examples: ``sumIf(column, cond)``, ``countIf(cond)``, ``avgIf(x, cond)``, ``quantilesTimingIf(level1, level2)(x, cond)``, ``argMinIf(arg, val, cond)`` and so on. - -You can use aggregate functions to calculate aggregates for multiple conditions at once, without using subqueries and JOINs. -For example, in Yandex.Metrica, we use conditional aggregate functions for implementing segment comparison functionality. - -Array combinator. Aggregate functions for array arguments ---------------------------------------------------------- -The -Array suffix can be appended to any aggregate function. In this case, the aggregate function takes arguments of the 'Array(T)' type (arrays) instead of 'T' type arguments. If the aggregate function accepts multiple arguments, this must be arrays of equal lengths. When processing arrays, the aggregate function works like the original aggregate function across all array elements. - -Example 1: ``sumArray(arr)`` - Totals all the elements of all 'arr' arrays. In this example, it could have been written more simply: sum(arraySum(arr)). - -Example 2: ``uniqArray(arr)`` - Count the number of unique elements in all 'arr' arrays. This could be done an easier way: ``uniq(arrayJoin(arr))``, but it's not always possible to add 'arrayJoin' to a query. - -The ``-If`` and ``-Array`` combinators can be used together. However, 'Array' must come first, then 'If'. -Examples: ``uniqArrayIf(arr, cond)``, ``quantilesTimingArrayIf(level1, level2)(arr, cond)``. Due to this order, the 'cond' argument can't be an array. - -State combinator ----------------- -If this combinator is used, the aggregate function returns intermediate aggregation state (for example, in the case of the ``uniqCombined`` function, a HyperLogLog structure for calculating the number of unique values), which has type of ``AggregateFunction(...)`` and can be used for further processing or can be saved to a table for subsequent pre-aggregation - see the sections "AggregatingMergeTree" and "functions for working with intermediate aggregation states". - -Merge combinator ----------------- -In the case of using this combinator, the aggregate function will take as an argument the intermediate state of an aggregation, pre-aggregate (combine together) these states, and return the finished/complete value. - -MergeState combinator ---------------------- -Merges the intermediate aggregation states, similar to the -Merge combinator, but returns a non-complete value, but an intermediate aggregation state, similar to the -State combinator. diff --git a/docs/en/agg_functions/parametric_functions.md b/docs/en/agg_functions/parametric_functions.md new file mode 100644 index 00000000000..8539312c9a1 --- /dev/null +++ b/docs/en/agg_functions/parametric_functions.md @@ -0,0 +1,72 @@ + + +# Parametric aggregate functions + +Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. + +## sequenceMatch(pattern)(time, cond1, cond2, ...) + +Pattern matching for event chains. + +`pattern` is a string containing a pattern to match. The pattern is similar to a regular expression. + +`time` is the time of the event with the DateTime type. + +`cond1`, `cond2` ... is from one to 32 arguments of type UInt8 that indicate whether a certain condition was met for the event. + +The function collects a sequence of events in RAM. Then it checks whether this sequence matches the pattern. +It returns UInt8: 0 if the pattern isn't matched, or 1 if it matches. + +Example: `sequenceMatch ('(?1).*(?2)')(EventTime, URL LIKE '%company%', URL LIKE '%cart%')` + +- whether there was a chain of events in which a pageview with 'company' in the address occurred earlier than a pageview with 'cart' in the address. + +This is a singular example. You could write it using other aggregate functions: + +```text +minIf(EventTime, URL LIKE '%company%') < maxIf(EventTime, URL LIKE '%cart%'). +``` + +However, there is no such solution for more complex situations. + +Pattern syntax: + +`(?1)` refers to the condition (any number can be used in place of 1). + +`.*` is any number of any events. + +`(?t>=1800)` is a time condition. + +Any quantity of any type of events is allowed over the specified time. + +Instead of `>=`, the following operators can be used:`<`, `>`, `<=`. + +Any number may be specified in place of 1800. + +Events that occur during the same second can be put in the chain in any order. This may affect the result of the function. + +## sequenceCount(pattern)(time, cond1, cond2, ...) + +Works the same way as the sequenceMatch function, but instead of returning whether there is an event chain, it returns UInt64 with the number of event chains found. +Chains are searched for without overlapping. In other words, the next chain can start only after the end of the previous one. + +## uniqUpTo(N)(x) + +Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. + +Recommended for use with small Ns, up to 10. The maximum value of N is 100. + +For the state of an aggregate function, it uses the amount of memory equal to 1 + N \* the size of one value of bytes. +For strings, it stores a non-cryptographic hash of 8 bytes. That is, the calculation is approximated for strings. + +The function also works for several arguments. + +It works as fast as possible, except for cases when a large N value is used and the number of unique values is slightly less than N. + +Usage example: + +```text +Problem: Generate a report that shows only keywords that produced at least 5 unique users. +Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 +``` + diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md new file mode 100644 index 00000000000..8f5bf0ed2d7 --- /dev/null +++ b/docs/en/agg_functions/reference.md @@ -0,0 +1,335 @@ + + +# Function reference + +## count() + +Counts the number of rows. Accepts zero arguments and returns UInt64. +The syntax `COUNT(DISTINCT x)` is not supported. The separate `uniq` aggregate function exists for this purpose. + +A `SELECT count() FROM table` query is not optimized, because the number of entries in the table is not stored separately. It will select some small column from the table and count the number of values in it. + +## any(x) + +Selects the first encountered value. +The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate. +To get a determinate result, you can use the 'min' or 'max' function instead of 'any'. + +In some cases, you can rely on the order of execution. This applies to cases when SELECT comes from a subquery that uses ORDER BY. + +When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function. + +## anyHeavy + +Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic. + +``` +anyHeavy(column) +``` + +**Arguments** + +- `column` – The column name. + +**Example** + +Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select any frequently occurring value in the `AirlineID` column. + +```sql +SELECT anyHeavy(AirlineID) AS res +FROM ontime +``` + +``` +┌───res─┐ +│ 19690 │ +└───────┘ +``` + +## anyLast(x) + +Selects the last value encountered. +The result is just as indeterminate as for the `any` function. + +## min(x) + +Calculates the minimum. + +## max(x) + +Calculates the maximum. + +## argMin(arg, val) + +Calculates the 'arg' value for a minimal 'val' value. If there are several different values of 'arg' for minimal values of 'val', the first of these values encountered is output. + +## argMax(arg, val) + +Calculates the 'arg' value for a maximum 'val' value. If there are several different values of 'arg' for maximum values of 'val', the first of these values encountered is output. + +## sum(x) + +Calculates the sum. +Only works for numbers. + +## sumWithOverflow(x) + +Computes the sum of the numbers, using the same data type for the result as for the input parameters. If the sum exceeds the maximum value for this data type, the function returns an error. + +Only works for numbers. + +## sumMap(key, value) + +Totals the 'value' array according to the keys specified in the 'key' array. +The number of elements in 'key' and 'value' must be the same for each row that is totaled. +Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. + +Example: + +```sql +CREATE TABLE sum_map( + date Date, + timeslot DateTime, + statusMap Nested( + status UInt16, + requests UInt64 + ) +) ENGINE = Log; +INSERT INTO sum_map VALUES + ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), + ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); +SELECT + timeslot, + sumMap(statusMap.status, statusMap.requests) +FROM sum_map +GROUP BY timeslot +``` + +```text +┌────────────timeslot─┬─sumMap(statusMap.status, statusMap.requests)─┐ +│ 2000-01-01 00:00:00 │ ([1,2,3,4,5],[10,10,20,10,10]) │ +│ 2000-01-01 00:01:00 │ ([4,5,6,7,8],[10,10,20,10,10]) │ +└─────────────────────┴──────────────────────────────────────────────┘ +``` + +## avg(x) + +Calculates the average. +Only works for numbers. +The result is always Float64. + +## uniq(x) + +Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments. + +Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65536. +This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions). + +The result is determinate (it doesn't depend on the order of query processing). + +## uniqCombined(x) + +Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments. + +A combination of three algorithms is used: array, hash table and [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) with an error correction table. The memory consumption is several times smaller than for the `uniq` function, and the accuracy is several times higher. Performance is slightly lower than for the `uniq` function, but sometimes it can be even higher than it, such as with distributed queries that transmit a large number of aggregation states over the network. The maximum state size is 96 KiB (HyperLogLog of 217 6-bit cells). + +The result is determinate (it doesn't depend on the order of query processing). + +The `uniqCombined` function is a good default choice for calculating the number of different values. + +## uniqHLL12(x) + +Uses the [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm to approximate the number of different values of the argument. +212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. + +The result is determinate (it doesn't depend on the order of query processing). + +In most cases, use the `uniq` or `uniqCombined` function. + +## uniqExact(x) + +Calculates the number of different values of the argument, exactly. +There is no reason to fear approximations. It's better to use the `uniq` function. +Use the `uniqExact` function if you definitely need an exact result. + +The `uniqExact` function uses more memory than the `uniq` function, because the size of the state has unbounded growth as the number of different values increases. + +## groupArray(x), groupArray(max_size)(x) + +Creates an array of argument values. +Values can be added to the array in any (indeterminate) order. + +The second version (with the `max_size` parameter) limits the size of the resulting array to `max_size` elements. +For example, `groupArray (1) (x)` is equivalent to `[any (x)]`. + +In some cases, you can still rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`. + + + +## groupArrayInsertAt + +Inserts a value into the array in the specified position. + +Accepts the value and position as input. If several values ​​are inserted into the same position, any of them might end up in the resulting array (the first one will be used in the case of single-threaded execution). If no value is inserted into a position, the position is assigned the default value. + +Optional parameters: + +- The default value for substituting in empty positions. +- The length of the resulting array. This allows you to receive arrays of the same size for all the aggregate keys. When using this parameter, the default value must be specified. + +## groupUniqArray(x) + +Creates an array from different argument values. Memory consumption is the same as for the `uniqExact` function. + +## quantile(level)(x) + +Approximates the 'level' quantile. 'level' is a constant, a floating-point number from 0 to 1. +We recommend using a 'level' value in the range of 0.01..0.99 +Don't use a 'level' value equal to 0 or 1 – use the 'min' and 'max' functions for these cases. + +In this function, as well as in all functions for calculating quantiles, the 'level' parameter can be omitted. In this case, it is assumed to be equal to 0.5 (in other words, the function will calculate the median). + +Works for numbers, dates, and dates with times. +Returns: for numbers – Float64; for dates – a date; for dates with times – a date with time. + +Uses [reservoir sampling](https://ru.wikipedia.org/wiki/Reservoir_sampling) with a reservoir size up to 8192. +If necessary, the result is output with linear approximation from the two neighboring values. +This algorithm provides very low accuracy. See also: `quantileTiming`, `quantileTDigest`, `quantileExact`. + +The result depends on the order of running the query, and is nondeterministic. + +When using multiple `quantile` (and similar) functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the `quantiles` (and similar) functions. + +## quantileDeterministic(level)(x, determinator) + +Works the same way as the `quantile` function, but the result is deterministic and does not depend on the order of query execution. + +To achieve this, the function takes a second argument – the "determinator". This is a number whose hash is used instead of a random number generator in the reservoir sampling algorithm. For the function to work correctly, the same determinator value should not occur too often. For the determinator, you can use an event ID, user ID, and so on. + +Don't use this function for calculating timings. There is a more suitable function for this purpose: `quantileTiming`. + +## quantileTiming(level)(x) + +Computes the quantile of 'level' with a fixed precision. +Works for numbers. Intended for calculating quantiles of page loading time in milliseconds. + +If the value is greater than 30,000 (a page loading time of more than 30 seconds), the result is equated to 30,000. + +If the total value is not more than about 5670, then the calculation is accurate. + +Otherwise: + +- if the time is less than 1024 ms, then the calculation is accurate. +- otherwise the calculation is rounded to a multiple of 16 ms. + +When passing negative values to the function, the behavior is undefined. + +The returned value has the Float32 type. If no values were passed to the function (when using `quantileTimingIf`), 'nan' is returned. The purpose of this is to differentiate these instances from zeros. See the note on sorting NaNs in "ORDER BY clause". + +The result is determinate (it doesn't depend on the order of query processing). + +For its purpose (calculating quantiles of page loading times), using this function is more effective and the result is more accurate than for the `quantile` function. + +## quantileTimingWeighted(level)(x, weight) + +Differs from the 'medianTiming' function in that it has a second argument, "weights". Weight is a non-negative integer. +The result is calculated as if the 'x' value were passed 'weight' number of times to the 'medianTiming\` function. + +## quantileExact(level)(x) + +Computes the quantile of 'level' exactly. To do this, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes O(n) memory, where 'n' is the number of values that were passed. However, for a small number of values, the function is very effective. + +## quantileExactWeighted(level)(x, weight) + +Computes the quantile of 'level' exactly. In addition, each value is counted with its weight, as if it is present 'weight' times. The arguments of the function can be considered as histograms, where the value 'x' corresponds to a histogram "column" of the height 'weight', and the function itself can be considered as a summation of histograms. + +A hash table is used as the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than `quantileExact`. You can use this function instead of `quantileExact` and specify the weight as 1. + +## quantileTDigest(level)(x) + +Approximates the quantile level using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm. The maximum error is 1%. Memory consumption by State is proportional to the logarithm of the number of passed values. + +The performance of the function is lower than for ` quantile`, ` quantileTiming`. In terms of the ratio of State size to precision, this function is much better than `quantile`. + +The result depends on the order of running the query, and is nondeterministic. + +## median + +All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical. + +## quantiles(level1, level2, ...)(x) + +All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. + +## varSamp(x) + +Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. + +It represents an unbiased estimate of the variance of a random variable, if the values passed to the function are a sample of this random amount. + +Returns `Float64`. When `n <= 1`, returns `+∞`. + +## varPop(x) + +Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. + +In other words, dispersion for a set of values. Returns `Float64`. + +## stddevSamp(x) + +The result is equal to the square root of `varSamp(x)`. + +## stddevPop(x) + +The result is equal to the square root of `varPop(x)`. + +## topK + +Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves). + +Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf). + +``` +topK(N)(column) +``` + +This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values. + +We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`. + +**Arguments** + +- 'N' – The number of values. +- ' x ' – The column. + +**Example** + +Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select the three most frequently occurring values in the `AirlineID` column. + +```sql +SELECT topK(3)(AirlineID) AS res +FROM ontime +``` + +``` +┌─res─────────────────┐ +│ [19393,19790,19805] │ +└─────────────────────┘ +``` + +## covarSamp(x, y) + +Calculates the value of `Σ((x - x̅)(y - y̅)) / (n - 1)`. + +Returns Float64. When `n <= 1`, returns +∞. + +## covarPop(x, y) + +Calculates the value of `Σ((x - x̅)(y - y̅)) / n`. + +## corr(x, y) + +Calculates the Pearson correlation coefficient: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. + diff --git a/docs/en/conf.py b/docs/en/conf.py index c58139ce779..12f14adfd15 100644 --- a/docs/en/conf.py +++ b/docs/en/conf.py @@ -17,6 +17,13 @@ import collections import os import sys +from recommonmark.parser import CommonMarkParser +from recommonmark.transform import AutoStructify + +source_parsers = { + '.md': CommonMarkParser, +} + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -41,7 +48,7 @@ templates_path = [ ] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = [ '.rst', '.md' ] # The encoding of source files. #source_encoding = 'utf-8-sig' @@ -299,3 +306,5 @@ def add_filters(app): def setup(app): app.add_javascript('custom.js') app.connect(str('builder-inited'), add_filters) + app.add_config_value('recommonmark_config', {'enable_eval_rst': True}, True) + app.add_transform(AutoStructify) diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md new file mode 100644 index 00000000000..6961cd19f12 --- /dev/null +++ b/docs/en/data_types/array.md @@ -0,0 +1,5 @@ +# Array(T) + +An array of elements of type T. The T type can be any type, including an array. +We don't recommend using multidimensional arrays, because they are not well supported (for example, you can't store multidimensional arrays in tables with a MergeTree engine). + diff --git a/docs/en/data_types/array.rst b/docs/en/data_types/array.rst deleted file mode 100644 index 159c2cd9a99..00000000000 --- a/docs/en/data_types/array.rst +++ /dev/null @@ -1,6 +0,0 @@ -Array(T) --------- - -Array of T-type items. The T type can be any type, including an array. - -We don't recommend using multidimensional arrays, because they are not well supported (for example, you can't store multidimensional arrays in tables with engines from MergeTree family). diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md new file mode 100644 index 00000000000..6ea8aefbe74 --- /dev/null +++ b/docs/en/data_types/boolean.md @@ -0,0 +1,4 @@ +# Boolean values + +There isn't a separate type for boolean values. They use the UInt8 type, restricted to the values 0 or 1. + diff --git a/docs/en/data_types/boolean.rst b/docs/en/data_types/boolean.rst deleted file mode 100644 index 501e7ce7b82..00000000000 --- a/docs/en/data_types/boolean.rst +++ /dev/null @@ -1,4 +0,0 @@ -Boolean -------- - -There is no separate type for boolean values. For them, the type UInt8 is used, in which only the values 0 and 1 are used. diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md new file mode 100644 index 00000000000..355e5555bfa --- /dev/null +++ b/docs/en/data_types/date.md @@ -0,0 +1,7 @@ +# Date + +Date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2038, but it may be expanded to 2106). +The minimum value is output as 0000-00-00. + +The date is stored without the time zone. + diff --git a/docs/en/data_types/date.rst b/docs/en/data_types/date.rst deleted file mode 100644 index 2df6fbf4174..00000000000 --- a/docs/en/data_types/date.rst +++ /dev/null @@ -1,7 +0,0 @@ -Date ----- - -A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage. Currently, this is until the year 2106 (the year 2106 is not covered). -The minimum value is output as 0000-00-00. - -The date is stored without the time zone. diff --git a/docs/en/data_types/datetime.rst b/docs/en/data_types/datetime.md similarity index 68% rename from docs/en/data_types/datetime.rst rename to docs/en/data_types/datetime.md index 7df690a2dbb..1be9ef867e1 100644 --- a/docs/en/data_types/datetime.rst +++ b/docs/en/data_types/datetime.md @@ -1,16 +1,15 @@ -DateTime --------- +# DateTime -Date with time. Stored in four bytes as a Unix timestamp (unsigned). Allows storing values in the same range as for the Date type. The minimal value is output as 0000-00-00 00:00:00. The time is stored with accuracy up to one second (without leap seconds). +Date with time. Stored in four bytes as a Unix timestamp (unsigned). Allows storing values in the same range as for the Date type. The minimal value is output as 0000-00-00 00:00:00. +The time is stored with accuracy up to one second (without leap seconds). - -Time zones -~~~~~~~~~~ +## Time zones The date with time is converted from text (divided into component parts) to binary and back, using the system's time zone at the time the client or server starts. In text format, information about daylight savings is lost. -Note that by default the client adopts the server time zone at the beginning of the session. You can change this behaviour with the --use_client_time_zone command line switch. +By default, the client switches to the timezone of the server when it connects. You can change this behavior by enabling the client command-line option `--use_client_time_zone`. Supports only those time zones that never had the time differ from UTC for a partial number of hours (without leap seconds) over the entire time range you will be working with. So when working with a textual date (for example, when saving text dumps), keep in mind that there may be ambiguity during changes for daylight savings time, and there may be problems matching data if the time zone changed. + diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md new file mode 100644 index 00000000000..c9c8af3d2ed --- /dev/null +++ b/docs/en/data_types/enum.md @@ -0,0 +1,31 @@ +# Enum + +Enum8 or Enum16. A finite set of string values that can be stored more efficiently than the `String` data type. + +Example: + +```text +Enum8('hello' = 1, 'world' = 2) +``` + +- A data type with two possible values: 'hello' and 'world'. + +Each of the values is assigned a number in the range `-128 ... 127` for `Enum8` or in the range `-32768 ... 32767` for `Enum16`. All the strings and numbers must be different. An empty string is allowed. If this type is specified (in a table definition), numbers can be in an arbitrary order. However, the order does not matter. + +In RAM, this type of column is stored in the same way as `Int8` or `Int16` of the corresponding numerical values. +When reading in text form, ClickHouse parses the value as a string and searches for the corresponding string from the set of Enum values. If it is not found, an exception is thrown. When reading in text format, the string is read and the corresponding numeric value is looked up. An exception will be thrown if it is not found. +When writing in text form, it writes the value as the corresponding string. If column data contains garbage (numbers that are not from the valid set), an exception is thrown. When reading and writing in binary form, it works the same way as for Int8 and Int16 data types. +The implicit default value is the value with the lowest number. + +During `ORDER BY`, `GROUP BY`, `IN`, `DISTINCT` and so on, Enums behave the same way as the corresponding numbers. For example, ORDER BY sorts them numerically. Equality and comparison operators work the same way on Enums as they do on the underlying numeric values. + +Enum values cannot be compared with numbers. Enums can be compared to a constant string. If the string compared to is not a valid value for the Enum, an exception will be thrown. The IN operator is supported with the Enum on the left hand side and a set of strings on the right hand side. The strings are the values of the corresponding Enum. + +Most numeric and string operations are not defined for Enum values, e.g. adding a number to an Enum or concatenating a string to an Enum. +However, the Enum has a natural `toString` function that returns its string value. + +Enum values are also convertible to numeric types using the `toT` function, where T is a numeric type. When T corresponds to the enum’s underlying numeric type, this conversion is zero-cost. +The Enum type can be changed without cost using ALTER, if only the set of values is changed. It is possible to both add and remove members of the Enum using ALTER (removing is safe only if the removed value has never been used in the table). As a safeguard, changing the numeric value of a previously defined Enum member will throw an exception. + +Using ALTER, it is possible to change an Enum8 to an Enum16 or vice versa, just like changing an Int8 to Int16. + diff --git a/docs/en/data_types/enum.rst b/docs/en/data_types/enum.rst deleted file mode 100644 index 6251b4db4c1..00000000000 --- a/docs/en/data_types/enum.rst +++ /dev/null @@ -1,30 +0,0 @@ -Enum ----- - -Enum8 or Enum16. A set of enumerated string values that are stored as Int8 or Int16. - -Example: - -.. code-block:: sql - - Enum8('hello' = 1, 'world' = 2) - -This data type has two possible values - 'hello' and 'world'. - -The numeric values must be within -128..127 for ``Enum8`` and -32768..32767 for ``Enum16``. Every member of the enum must also have different numbers. The empty string is a valid value. The numbers do not need to be sequential and can be in any order. The order does not matter. - -In memory, the data is stored in the same way as the numeric types ``Int8`` and ``Int16``. -When reading in text format, the string is read and the corresponding numeric value is looked up. An exception will be thrown if it is not found. -When writing in text format, the stored number is looked up and the corresponding string is written out. An exception will be thrown if the number does not correspond to a known value. -In binary format, the information is saved in the same way as ``Int8`` and ``Int16``. -The implicit default value for an Enum is the value having the smallest numeric value. - -In ORDER BY, GROUP BY, IN, DISTINCT, etc. Enums behave like the numeric value. e.g. they will be sorted by the numeric value in an ``ORDER BY``. Equality and comparison operators behave like they do on the underlying numeric value. - -Enum values cannot be compared to numbers, they must be compared to a string. If the string compared to is not a valid value for the Enum, an exception will be thrown. The ``IN`` operator is supported with the Enum on the left hand side and a set of strings on the right hand side. - -Most numeric and string operations are not defined for Enum values, e.g. adding a number to an Enum or concatenating a string to an Enum. However, the toString function can be used to convert the Enum to its string value. Enum values are also convertible to numeric types using the ``toT`` function where ``T`` is a numeric type. When ``T`` corresponds to the enum's underlying numeric type, this conversion is zero-cost. - -It is possible to add new members to the ``Enum`` using ``ALTER``. If the only change is to the set of values, the operation will be almost instant. It is also possible to remove members of the Enum using ALTER. Removing members is only safe if the removed value has never been used in the table. As a safeguard, changing the numeric value of a previously defined Enum member will throw an exception. - -Using ``ALTER``, it is possible to change an ``Enum8`` to an ``Enum16`` or vice versa - just like changing an ``Int8`` to ``Int16``. diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md new file mode 100644 index 00000000000..d83b0087057 --- /dev/null +++ b/docs/en/data_types/fixedstring.md @@ -0,0 +1,10 @@ +# FixedString(N) + +A fixed-length string of N bytes (not characters or code points). N must be a strictly positive natural number. +When the server reads a string that contains fewer bytes (such as when parsing INSERT data), the string is padded to N bytes by appending null bytes at the right. +When the server reads a string that contains more bytes, an error message is returned. +When the server writes a string (such as when outputting the result of a SELECT query), null bytes are not trimmed off of the end of the string, but are output. +Note that this behavior differs from MySQL behavior for the CHAR type (where strings are padded with spaces, and the spaces are removed for output). + +Fewer functions can work with the FixedString(N) type than with String, so it is less convenient to use. + diff --git a/docs/en/data_types/fixedstring.rst b/docs/en/data_types/fixedstring.rst deleted file mode 100644 index c58adb1ba13..00000000000 --- a/docs/en/data_types/fixedstring.rst +++ /dev/null @@ -1,10 +0,0 @@ -FixedString(N) --------------- - -A fixed-length string of N bytes (not characters or code points). N must be a strictly positive natural number. -When server reads a string (as an input passed in INSERT query, for example) that contains fewer bytes, the string is padded to N bytes by appending null bytes at the right. -When server reads a string that contains more bytes, an error message is returned. -When server writes a string (as an output of SELECT query, for example), null bytes are not trimmed off of the end of the string, but are output. -Note that this behavior differs from MySQL behavior for the CHAR type (where strings are padded with spaces, and the spaces are removed for output). - -Fewer functions can work with the FixedString(N) type than with String, so it is less convenient to use. diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md new file mode 100644 index 00000000000..eff0559b75e --- /dev/null +++ b/docs/en/data_types/float.md @@ -0,0 +1,71 @@ +# Float32, Float64 + +[Floating point numbers](https://en.wikipedia.org/wiki/IEEE_754). + +Types are equivalent to types of C: + +- `Float32` - `float` +- `Float64` - ` double` + +We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds. + +## Using floating-point numbers + +- Computations with floating-point numbers might produce a rounding error. + + ```sql + SELECT 1 - 0.9 + ``` + + ``` + ┌───────minus(1, 0.9)─┐ + │ 0.09999999999999998 │ + └─────────────────────┘ + ``` + +- The result of the calculation depends on the calculation method (the processor type and architecture of the computer system). + +- Floating-point calculations might result in numbers such as infinity (`Inf`) and "not-a-number" (`NaN`). This should be taken into account when processing the results of calculations. + +- When reading floating point numbers from rows, the result might not be the nearest machine-representable number. + +## NaN and Inf + +In contrast to standard SQL, ClickHouse supports the following categories of floating-point numbers: + +- `Inf` – Infinity. + + ```sql + SELECT 0.5 / 0 + ``` + + ``` + ┌─divide(0.5, 0)─┐ + │ inf │ + └────────────────┘ + ``` +- `-Inf` – Negative infinity. + + ```sql + SELECT -0.5 / 0 + ``` + + ``` + ┌─divide(-0.5, 0)─┐ + │ -inf │ + └─────────────────┘ + ``` +- `NaN` – Not a number. + + ``` + SELECT 0 / 0 + ``` + + ``` + ┌─divide(0, 0)─┐ + │ nan │ + └──────────────┘ + ``` + + See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by). + diff --git a/docs/en/data_types/float.rst b/docs/en/data_types/float.rst deleted file mode 100644 index 8259f221813..00000000000 --- a/docs/en/data_types/float.rst +++ /dev/null @@ -1,7 +0,0 @@ -Float32, Float64 ----------------- - -Floating-point numbers are just like 'float' and 'double' in the C language. -In contrast to standard SQL, floating-point numbers support 'inf', '-inf', and even 'nan's. -See the notes on sorting nans in "ORDER BY clause". -We do not recommend storing floating-point numbers in tables. diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md new file mode 100644 index 00000000000..4957f9c6c8c --- /dev/null +++ b/docs/en/data_types/index.md @@ -0,0 +1,13 @@ + + +# Data types + +```eval_rst +.. toctree:: + :glob: + + * + */index + +``` + diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md new file mode 100644 index 00000000000..c8e31249854 --- /dev/null +++ b/docs/en/data_types/int_uint.md @@ -0,0 +1,18 @@ +# UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 + +Fixed-length integers, with or without a sign. + +## Int ranges + +- Int8 - [-128 : 127] +- Int16 - [-32768 : 32767] +- Int32 - [-2147483648 : 2147483647] +- Int64 - [-9223372036854775808 : 9223372036854775807] + +## Uint ranges + +- UInt8 - [0 : 255] +- UInt16 - [0 : 65535] +- UInt32 - [0 : 4294967295] +- UInt64 - [0 : 18446744073709551615] + diff --git a/docs/en/data_types/int_uint.rst b/docs/en/data_types/int_uint.rst deleted file mode 100644 index 7509d3ba099..00000000000 --- a/docs/en/data_types/int_uint.rst +++ /dev/null @@ -1,40 +0,0 @@ -UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 --------------------------------------------------------- - -Fixed-length integers, with or without a sign. - -Int ranges -"""""""""" - -.. table:: - - +--------+----------------------+-----------------------+ - | Тип | From | To | - +========+======================+=======================+ - | Int8 | -128 | 127 | - +--------+----------------------+-----------------------+ - | Int16 | -32768 | 32767 | - +--------+----------------------+-----------------------+ - | Int32 | -2147483648 | 2147483647 | - +--------+----------------------+-----------------------+ - | Int64 | -9223372036854775808 | 9223372036854775807 | - +--------+----------------------+-----------------------+ - - - -Uint ranges -""""""""""" - -.. table:: - - +--------+----------------------+-----------------------+ - | Тип | From | To | - +========+======================+=======================+ - | UInt8 | 0 | 255 | - +--------+----------------------+-----------------------+ - | UInt16 | 0 | 65535 | - +--------+----------------------+-----------------------+ - | UInt32 | 0 | 4294967295 | - +--------+----------------------+-----------------------+ - | UInt64 | 0 | 18446744073709551615 | - +--------+----------------------+-----------------------+ diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.rst b/docs/en/data_types/nested_data_structures/aggregatefunction.md similarity index 63% rename from docs/en/data_types/nested_data_structures/aggregatefunction.rst rename to docs/en/data_types/nested_data_structures/aggregatefunction.md index 1b756a63ef5..33e7b4f316c 100644 --- a/docs/en/data_types/nested_data_structures/aggregatefunction.rst +++ b/docs/en/data_types/nested_data_structures/aggregatefunction.md @@ -1,4 +1,4 @@ -AggregateFunction(name, types_of_arguments...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# AggregateFunction(name, types_of_arguments...) The intermediate state of an aggregate function. To get it, use aggregate functions with the '-State' suffix. For more information, see "AggregatingMergeTree". + diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md new file mode 100644 index 00000000000..16317c4fb46 --- /dev/null +++ b/docs/en/data_types/nested_data_structures/index.md @@ -0,0 +1,9 @@ +# Nested data structures + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/data_types/nested_data_structures/index.rst b/docs/en/data_types/nested_data_structures/index.rst deleted file mode 100644 index f302eb80ca6..00000000000 --- a/docs/en/data_types/nested_data_structures/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Nested data structures ----------------------- - -.. toctree:: - :glob: - - * diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md new file mode 100644 index 00000000000..8a4bd4297fb --- /dev/null +++ b/docs/en/data_types/nested_data_structures/nested.md @@ -0,0 +1,98 @@ +# Nested(Name1 Type1, Name2 Type2, ...) + +A nested data structure is like a nested table. The parameters of a nested data structure – the column names and types – are specified the same way as in a CREATE query. Each table row can correspond to any number of rows in a nested data structure. + +Example: + +```sql +CREATE TABLE test.visits +( + CounterID UInt32, + StartDate Date, + Sign Int8, + IsNew UInt8, + VisitID UInt64, + UserID UInt64, + ... + Goals Nested + ( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32 + ), + ... +) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign) +``` + +This example declares the `Goals` nested data structure, which contains data about conversions (goals reached). Each row in the 'visits' table can correspond to zero or any number of conversions. + +Only a single nesting level is supported. Columns of nested structures containing arrays are equivalent to multidimensional arrays, so they have limited support (there is no support for storing these columns in tables with the MergeTree engine). + +In most cases, when working with a nested data structure, its individual columns are specified. To do this, the column names are separated by a dot. These columns make up an array of matching types. All the column arrays of a single nested data structure have the same length. + +Example: + +```sql +SELECT + Goals.ID, + Goals.EventTime +FROM test.visits +WHERE CounterID = 101500 AND length(Goals.ID) < 5 +LIMIT 10 +``` + +```text +┌─Goals.ID───────────────────────┬─Goals.EventTime───────────────────────────────────────────────────────────────────────────┐ +│ [1073752,591325,591325] │ ['2014-03-17 16:38:10','2014-03-17 16:38:48','2014-03-17 16:42:27'] │ +│ [1073752] │ ['2014-03-17 00:28:25'] │ +│ [1073752] │ ['2014-03-17 10:46:20'] │ +│ [1073752,591325,591325,591325] │ ['2014-03-17 13:59:20','2014-03-17 22:17:55','2014-03-17 22:18:07','2014-03-17 22:18:51'] │ +│ [] │ [] │ +│ [1073752,591325,591325] │ ['2014-03-17 11:37:06','2014-03-17 14:07:47','2014-03-17 14:36:21'] │ +│ [] │ [] │ +│ [] │ [] │ +│ [591325,1073752] │ ['2014-03-17 00:46:05','2014-03-17 00:46:05'] │ +│ [1073752,591325,591325,591325] │ ['2014-03-17 13:28:33','2014-03-17 13:30:26','2014-03-17 18:51:21','2014-03-17 18:51:45'] │ +└────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +It is easiest to think of a nested data structure as a set of multiple column arrays of the same length. + +The only place where a SELECT query can specify the name of an entire nested data structure instead of individual columns is the ARRAY JOIN clause. For more information, see "ARRAY JOIN clause". Example: + +```sql +SELECT + Goal.ID, + Goal.EventTime +FROM test.visits +ARRAY JOIN Goals AS Goal +WHERE CounterID = 101500 AND length(Goals.ID) < 5 +LIMIT 10 +``` + +```text +┌─Goal.ID─┬──────Goal.EventTime─┐ +│ 1073752 │ 2014-03-17 16:38:10 │ +│ 591325 │ 2014-03-17 16:38:48 │ +│ 591325 │ 2014-03-17 16:42:27 │ +│ 1073752 │ 2014-03-17 00:28:25 │ +│ 1073752 │ 2014-03-17 10:46:20 │ +│ 1073752 │ 2014-03-17 13:59:20 │ +│ 591325 │ 2014-03-17 22:17:55 │ +│ 591325 │ 2014-03-17 22:18:07 │ +│ 591325 │ 2014-03-17 22:18:51 │ +│ 1073752 │ 2014-03-17 11:37:06 │ +└─────────┴─────────────────────┘ +``` + +You can't perform SELECT for an entire nested data structure. You can only explicitly list individual columns that are part of it. + +For an INSERT query, you should pass all the component column arrays of a nested data structure separately (as if they were individual column arrays). During insertion, the system checks that they have the same length. + +For a DESCRIBE query, the columns in a nested data structure are listed separately in the same way. + +The ALTER query is very limited for elements in a nested data structure. + diff --git a/docs/en/data_types/nested_data_structures/nested.rst b/docs/en/data_types/nested_data_structures/nested.rst deleted file mode 100644 index 5f30a0af8a0..00000000000 --- a/docs/en/data_types/nested_data_structures/nested.rst +++ /dev/null @@ -1,99 +0,0 @@ -Nested(Name1 Type1, Name2 Type2, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A nested data structure is like a nested table. The parameters of a nested data structure - the column names and types - are specified the same way as in a CREATE query. Each table row can correspond to any number of rows in a nested data structure. - -Example: - -.. code-block:: sql - - CREATE TABLE test.visits - ( - CounterID UInt32, - StartDate Date, - Sign Int8, - IsNew UInt8, - VisitID UInt64, - UserID UInt64, - ... - Goals Nested - ( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32 - ), - ... - ) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign) - -This example declares the 'Goals' nested data structure, which contains data about conversions (goals reached). Each row in the 'visits' table can correspond to zero or any number of conversions. - -Only a single nesting level is supported. Nested structure columns with array type are equivalent to multidimensional arrays and thus their support is limited (storing such columns in tables with engines from MergeTree family is not supported). - -In most cases, when working with a nested data structure, its individual columns are specified. To do this, the column names are separated by a dot. These columns make up an array of matching types. All the column arrays of a single nested data structure have the same length. - -Example: - -.. code-block:: sql - - SELECT - Goals.ID, - Goals.EventTime - FROM test.visits - WHERE CounterID = 101500 AND length(Goals.ID) < 5 - LIMIT 10 - -.. code-block:: text - - ┌─Goals.ID───────────────────────┬─Goals.EventTime───────────────────────────────────────────────────────────────────────────┐ - │ [1073752,591325,591325] │ ['2014-03-17 16:38:10','2014-03-17 16:38:48','2014-03-17 16:42:27'] │ - │ [1073752] │ ['2014-03-17 00:28:25'] │ - │ [1073752] │ ['2014-03-17 10:46:20'] │ - │ [1073752,591325,591325,591325] │ ['2014-03-17 13:59:20','2014-03-17 22:17:55','2014-03-17 22:18:07','2014-03-17 22:18:51'] │ - │ [] │ [] │ - │ [1073752,591325,591325] │ ['2014-03-17 11:37:06','2014-03-17 14:07:47','2014-03-17 14:36:21'] │ - │ [] │ [] │ - │ [] │ [] │ - │ [591325,1073752] │ ['2014-03-17 00:46:05','2014-03-17 00:46:05'] │ - │ [1073752,591325,591325,591325] │ ['2014-03-17 13:28:33','2014-03-17 13:30:26','2014-03-17 18:51:21','2014-03-17 18:51:45'] │ - └────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────┘ - -It is easiest to think of a nested data structure as a set of multiple column arrays of the same length. - -The only place where a SELECT query can specify the name of an entire nested data structure instead of individual columns is the ARRAY JOIN clause. For more information, see "ARRAY JOIN clause". Example: - -.. code-block:: sql - - SELECT - Goal.ID, - Goal.EventTime - FROM test.visits - ARRAY JOIN Goals AS Goal - WHERE CounterID = 101500 AND length(Goals.ID) < 5 - LIMIT 10 - -.. code-block:: text - - ┌─Goal.ID─┬──────Goal.EventTime─┐ - │ 1073752 │ 2014-03-17 16:38:10 │ - │ 591325 │ 2014-03-17 16:38:48 │ - │ 591325 │ 2014-03-17 16:42:27 │ - │ 1073752 │ 2014-03-17 00:28:25 │ - │ 1073752 │ 2014-03-17 10:46:20 │ - │ 1073752 │ 2014-03-17 13:59:20 │ - │ 591325 │ 2014-03-17 22:17:55 │ - │ 591325 │ 2014-03-17 22:18:07 │ - │ 591325 │ 2014-03-17 22:18:51 │ - │ 1073752 │ 2014-03-17 11:37:06 │ - └─────────┴─────────────────────┘ - - -You can't perform SELECT for an entire nested data structure. You can only explicitly list individual columns that are part of it. - -For an INSERT query, you should pass all the component column arrays of a nested data structure separately (as if they were individual column arrays). During insertion, the system checks that they have the same length. - -For a DESCRIBE query, the columns in a nested data structure are listed separately in the same way. - -The ALTER query is very limited for elements in a nested data structure. diff --git a/docs/en/data_types/special_data_types/expression.rst b/docs/en/data_types/special_data_types/expression.md similarity index 75% rename from docs/en/data_types/special_data_types/expression.rst rename to docs/en/data_types/special_data_types/expression.md index 3cd57b38746..0fec4b71578 100644 --- a/docs/en/data_types/special_data_types/expression.rst +++ b/docs/en/data_types/special_data_types/expression.md @@ -1,4 +1,4 @@ -Expression -~~~~~~~~~~ +# Expression Used for representing lambda expressions in high-order functions. + diff --git a/docs/en/data_types/special_data_types/index.rst b/docs/en/data_types/special_data_types/index.md similarity index 81% rename from docs/en/data_types/special_data_types/index.rst rename to docs/en/data_types/special_data_types/index.md index b17cb4f133c..8c575a42ad2 100644 --- a/docs/en/data_types/special_data_types/index.rst +++ b/docs/en/data_types/special_data_types/index.md @@ -1,9 +1,11 @@ -Special data types ------------------- +# Special data types Special data type values can't be saved to a table or output in results, but are used as the intermediate result of running a query. +```eval_rst .. toctree:: :glob: * +``` + diff --git a/docs/en/data_types/special_data_types/set.rst b/docs/en/data_types/special_data_types/set.md similarity index 85% rename from docs/en/data_types/special_data_types/set.rst rename to docs/en/data_types/special_data_types/set.md index 0a48c339984..346fe51e2bb 100644 --- a/docs/en/data_types/special_data_types/set.rst +++ b/docs/en/data_types/special_data_types/set.md @@ -1,4 +1,4 @@ -Set -~~~ +# Set Used for the right half of an IN expression. + diff --git a/docs/en/data_types/string.rst b/docs/en/data_types/string.md similarity index 96% rename from docs/en/data_types/string.rst rename to docs/en/data_types/string.md index 410b8b2586b..c2bffefe79d 100644 --- a/docs/en/data_types/string.rst +++ b/docs/en/data_types/string.md @@ -1,14 +1,12 @@ -String ------- +# String Strings of an arbitrary length. The length is not limited. The value can contain an arbitrary set of bytes, including null bytes. The String type replaces the types VARCHAR, BLOB, CLOB, and others from other DBMSs. - -Encodings -~~~~~~~~~ +## Encodings ClickHouse doesn't have the concept of encodings. Strings can contain an arbitrary set of bytes, which are stored and output as-is. If you need to store texts, we recommend using UTF-8 encoding. At the very least, if your terminal uses UTF-8 (as recommended), you can read and write your values without making conversions. Similarly, certain functions for working with strings have separate variations that work under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. For example, the 'length' function calculates the string length in bytes, while the 'lengthUTF8' function calculates the string length in Unicode code points, assuming that the value is UTF-8 encoded. + diff --git a/docs/en/data_types/tuple.rst b/docs/en/data_types/tuple.md similarity index 66% rename from docs/en/data_types/tuple.rst rename to docs/en/data_types/tuple.md index d53cd8daa14..5c996da7dbd 100644 --- a/docs/en/data_types/tuple.rst +++ b/docs/en/data_types/tuple.md @@ -1,6 +1,6 @@ -Tuple(T1, T2, ...) ------------------- +# Tuple(T1, T2, ...) Tuples can't be written to tables (other than Memory tables). They are used for temporary column grouping. Columns can be grouped when an IN expression is used in a query, and for specifying certain formal parameters of lambda functions. For more information, see "IN operators" and "Higher order functions". -Tuples can be output as the result of running a query. In this case, for text formats other than JSON*, values are comma-separated in brackets. In JSON* formats, tuples are output as arrays (in square brackets). +Tuples can be output as the result of running a query. In this case, for text formats other than JSON\*, values are comma-separated in brackets. In JSON\* formats, tuples are output as arrays (in square brackets). + diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md new file mode 100644 index 00000000000..54579ef30a2 --- /dev/null +++ b/docs/en/development/architecture.md @@ -0,0 +1,195 @@ +# Overview of ClickHouse architecture + +ClickHouse is a true column-oriented DBMS. Data is stored by columns, and during the execution of arrays (vectors or chunks of columns). Whenever possible, operations are dispatched on arrays, rather than on individual values. This is called "vectorized query execution," and it helps lower the cost of actual data processing. + +> This idea is nothing new. It dates back to the `APL` programming language and its descendants: `A +`, `J`, `K`, and `Q`. Array programming is used in scientific data processing. Neither is this idea something new in relational databases: for example, it is used in the `Vectorwise` system. + +There are two different approaches for speeding up the query processing: vectorized query execution and runtime code generation. In the latter, the code is generated for every kind of query on the fly, removing all indirection and dynamic dispatch. Neither of these approaches is strictly better than the other. Runtime code generation can be better when it's fuses many operations together, thus fully utilizing CPU execution units and the pipeline. Vectorized query execution can be less practical, because it involves the temporary vectors that must be written to the cache and read back. If the temporary data does not fit in the L2 cache, this becomes an issue. But vectorized query execution more easily utilizes the SIMD capabilities of the CPU. A [research paper](http://15721.courses.cs.cmu.edu/spring2016/papers/p5-sompolski.pdf) written by our friends shows that it is better to combine both approaches. ClickHouse uses vectorized query execution and has limited initial support for runtime code. + +## Columns + +To represent columns in memory (actually, chunks of columns), the `IColumn` interface is used. This interface provides helper methods for implementation of various relational operators. Almost all operations are immutable: they do not modify the original column, but create a new modified one. For example, the `IColumn :: filter` method accepts a filter byte mask. It is used for the `WHERE` and `HAVING` relational operators. Additional examples: the `IColumn :: permute` method to support `ORDER BY`, the `IColumn :: cut` method to support `LIMIT`, and so on. + +Various `IColumn` implementations (`ColumnUInt8`, `ColumnString` and so on) are responsible for the memory layout of columns. Memory layout is usually a contiguous array. For the integer type of columns it is just one contiguous array, like `std :: vector`. For `String` and `Array` columns, it is two vectors: one for all array elements, placed contiguously, and a second one for offsets to the beginning of each array. There is also `ColumnConst` that stores just one value in memory, but looks like a column. + +## Field + +Nevertheless, it is possible to work with individual values as well. To represent an individual value, the `Field` is used. `Field` is just a discriminated union of `UInt64`, `Int64`, `Float64`, `String` and `Array`. `IColumn` has the `operator[]` method to get the n-th value as a `Field`, and the `insert` method to append a `Field` to the end of a column. These methods are not very efficient, because they require dealing with temporary `Field` objects representing an individual value. There are more efficient methods, such as `insertFrom`, `insertRangeFrom`, and so on. + +`Field` doesn't have enough information about a specific data type for a table. For example, `UInt8`, `UInt16`, `UInt32`, and `UInt64` are all represented as `UInt64` in a `Field`. + +## Leaky abstractions + +`IColumn` has methods for common relational transformations of data, but they don't meet all needs. For example, `ColumnUInt64` doesn't have a method to calculate the sum of two columns, and `ColumnString` doesn't have a method to run a substring search. These countless routines are implemented outside of `IColumn`. + +Various functions on columns can be implemented in a generic, non-efficient way using `IColumn` methods to extract `Field` values, or in a specialized way using knowledge of inner memory layout of data in a specific `IColumn` implementation. To do this, functions are cast to a specific `IColumn` type and deal with internal representation directly. For example, `ColumnUInt64` has the `getData` method that returns a reference to an internal array, then a separate routine reads or fills that array directly. In fact, we have "leaky abstractions" to allow efficient specializations of various routines. + +## Data types + +`IDataType` is responsible for serialization and deserialization: for reading and writing chunks of columns or individual values in binary or text form. +`IDataType` directly corresponds to data types in tables. For example, there are `DataTypeUInt32`, `DataTypeDateTime`, `DataTypeString` and so on. + +`IDataType` and `IColumn` are only loosely related to each other. Different data types can be represented in memory by the same `IColumn` implementations. For example, `DataTypeUInt32` and `DataTypeDateTime` are both represented by `ColumnUInt32` or `ColumnConstUInt32`. In addition, the same data type can be represented by different `IColumn` implementations. For example, `DataTypeUInt8` can be represented by `ColumnUInt8` or `ColumnConstUInt8`. + +`IDataType` only stores metadata. For instance, `DataTypeUInt8` doesn't store anything at all (except vptr) and `DataTypeFixedString` stores just `N` (the size of fixed-size strings). + +`IDataType` has helper methods for various data formats. Examples are methods to serialize a value with possible quoting, to serialize a value for JSON, and to serialize a value as part of XML format. There is no direct correspondence to data formats. For example, the different data formats `Pretty` and `TabSeparated` can use the same `serializeTextEscaped` helper method from the `IDataType` interface. + +## Block + +A `Block` is a container that represents a subset (chunk) of a table in memory. It is just a set of triples: `(IColumn, IDataType, column name)`. During query execution, data is processed by `Block`s. If we have a `Block`, we have data (in the `IColumn` object), we have information about its type (in `IDataType`) that tells us how to deal with that column, and we have the column name (either the original column name from the table, or some artificial name assigned for getting temporary results of calculations). + +When we calculate some function over columns in a block, we add another column with its result to the block, and we don't touch columns for arguments of the function because operations are immutable. Later, unneeded columns can be removed from the block, but not modified. This is convenient for elimination of common subexpressions. + +Blocks are created for every processed chunk of data. Note that for the same type of calculation, the column names and types remain the same for different blocks, and only column data changes. It is better to split block data from the block header, because small block sizes will have a high overhead of temporary strings for copying shared_ptrs and column names. + +## Block Streams + +Block streams are for processing data. We use streams of blocks to read data from somewhere, perform data transformations, or write data to somewhere. `IBlockInputStream` has the `read` method to fetch the next block while available. `IBlockOutputStream` has the `write` method to push the block somewhere. + +Streams are responsible for: + +1. Reading or writing to a table. The table just returns a stream for reading or writing blocks. +2. Implementing data formats. For example, if you want to output data to a terminal in `Pretty` format, you create a block output stream where you push blocks, and it formats them. +3. Performing data transformations. Let's say you have `IBlockInputStream` and want to create a filtered stream. You create `FilterBlockInputStream` and initialize it with your stream. Then when you pull a block from `FilterBlockInputStream`, it pulls a block from your stream, filters it, and returns the filtered block to you. Query execution pipelines are represented this way. + +There are more sophisticated transformations. For example, when you pull from `AggregatingBlockInputStream`, it reads all data from its source, aggregates it, and then returns a stream of aggregated data for you. Another example: `UnionBlockInputStream` accepts many input sources in the constructor and also a number of threads. It launches multiple threads and reads from multiple sources in parallel. + +> Block streams use the "pull" approach to control flow: when you pull a block from the first stream, it consequently pulls the required blocks from nested streams, and the entire execution pipeline will work. Neither "pull" nor "push" is the best solution, because control flow is implicit, and that limits implementation of various features like simultaneous execution of multiple queries (merging many pipelines together). This limitation could be overcome with coroutines or just running extra threads that wait for each other. We may have more possibilities if we make control flow explicit: if we locate the logic for passing data from one calculation unit to another outside of those calculation units. Read this [article](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) for more thoughts. + +We should note that the query execution pipeline creates temporary data at each step. We try to keep block size small enough so that temporary data fits in the CPU cache. With that assumption, writing and reading temporary data is almost free in comparison with other calculations. We could consider an alternative, which is to fuse many operations in the pipeline together, to make the pipeline as short as possible and remove much of the temporary data. This could be an advantage, but it also has drawbacks. For example, a split pipeline makes it easy to implement caching intermediate data, stealing intermediate data from similar queries running at the same time, and merging pipelines for similar queries. + +## Formats + +Data formats are implemented with block streams. There are "presentational" formats only suitable for output of data to the client, such as `Pretty` format, which provides only `IBlockOutputStream`. And there are input/output formats, such as `TabSeparated` or `JSONEachRow`. + +There are also row streams: `IRowInputStream` and `IRowOutputStream`. They allow you to pull/push data by individual rows, not by blocks. And they are only needed to simplify implementation of row-oriented formats. The wrappers `BlockInputStreamFromRowInputStream` and `BlockOutputStreamFromRowOutputStream` allow you to convert row-oriented streams to regular block-oriented streams. + +## I/O + +For byte-oriented input/output, there are `ReadBuffer` and `WriteBuffer` abstract classes. They are used instead of C++ `iostream`'s. Don't worry: every mature C++ project is using something other than `iostream`'s for good reasons. + +`ReadBuffer` and `WriteBuffer` are just a contiguous buffer and a cursor pointing to the position in that buffer. Implementations may own or not own the memory for the buffer. There is a virtual method to fill the buffer with the following data (for `ReadBuffer`) or to flush the buffer somewhere (for `WriteBuffer`). The virtual methods are rarely called. + +Implementations of `ReadBuffer`/`WriteBuffer` are used for working with files and file descriptors and network sockets, for implementing compression (`CompressedWriteBuffer` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names `ConcatReadBuffer`, `LimitReadBuffer`, and `HashingWriteBuffer` speak for themselves. + +Read/WriteBuffers only deal with bytes. To help with formatted input/output (for instance, to write a number in decimal format), there are functions from `ReadHelpers` and `WriteHelpers` header files. + +Let's look at what happens when you want to write a result set in `JSON` format to stdout. You have a result set ready to be fetched from `IBlockInputStream`. You create `WriteBufferFromFileDescriptor(STDOUT_FILENO)` to write bytes to stdout. You create `JSONRowOutputStream`, initialized with that `WriteBuffer`, to write rows in `JSON` to stdout. You create `BlockOutputStreamFromRowOutputStream` on top of it, to represent it as `IBlockOutputStream`. Then you call `copyData` to transfer data from `IBlockInputStream` to `IBlockOutputStream`, and everything works. Internally, `JSONRowOutputStream` will write various JSON delimiters and call the `IDataType::serializeTextJSON` method with a reference to `IColumn` and the row number as arguments. Consequently, `IDataType::serializeTextJSON` will call a method from `WriteHelpers.h`: for example, `writeText` for numeric types and `writeJSONString` for `DataTypeString`. + +## Tables + +Tables are represented by the `IStorage` interface. Different implementations of that interface are different table engines. Examples are `StorageMergeTree`, `StorageMemory`, and so on. Instances of these classes are just tables. + +The most important `IStorage` methods are `read` and `write`. There are also `alter`, `rename`, `drop`, and so on. The `read` method accepts the following arguments: the set of columns to read from a table, the `AST` query to consider, and the desired number of streams to return. It returns one or multiple `IBlockInputStream` objects and information about the stage of data processing that was completed inside a table engine during query execution. + +In most cases, the read method is only responsible for reading the specified columns from a table, not for any further data processing. All further data processing is done by the query interpreter and is outside the responsibility of `IStorage`. + +But there are notable exceptions: + +- The AST query is passed to the `read` method and the table engine can use it to derive index usage and to read less data from a table. +- Sometimes the table engine can process data itself to a specific stage. For example, `StorageDistributed` can send a query to remote servers, ask them to process data to a stage where data from different remote servers can be merged, and return that preprocessed data. +The query interpreter then finishes processing the data. + +The table's `read` method can return multiple `IBlockInputStream` objects to allow parallel data processing. These multiple block input streams can read from a table in parallel. Then you can wrap these streams with various transformations (such as expression evaluation or filtering) that can be calculated independently and create a `UnionBlockInputStream` on top of them, to read from multiple streams in parallel. + +There are also `TableFunction`s. These are functions that return a temporary `IStorage` object to use in the `FROM` clause of a query. + +To get a quick idea of how to implement your own table engine, look at something simple, like `StorageMemory` or `StorageTinyLog`. + +> As the result of the `read` method, `IStorage` returns `QueryProcessingStage` – information about what parts of the query were already calculated inside storage. Currently we have only very coarse granularity for that information. There is no way for the storage to say "I have already processed this part of the expression in WHERE, for this range of data". We need to work on that. + +## Parsers + +A query is parsed by a hand-written recursive descent parser. For example, `ParserSelectQuery` just recursively calls the underlying parsers for various parts of the query. Parsers create an `AST`. The `AST` is represented by nodes, which are instances of `IAST`. + +> Parser generators are not used for historical reasons. + +## Interpreters + +Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery`and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to; and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time. + +`InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations or query. + +## Functions + +There are ordinary functions and aggregate functions. For aggregate functions, see the next section. + +Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`'s of data to implement vectorized query execution. + +There are some miscellaneous functions, like `blockSize`, `rowNumberInBlock`, and `runningAccumulate`, that exploit block processing and violate the independence of rows. + +ClickHouse has strong typing, so implicit type conversion doesn't occur. If a function doesn't support a specific combination of types, an exception will be thrown. But functions can work (be overloaded) for many different combinations of types. For example, the `plus` function (to implement the `+` operator) works for any combination of numeric types: `UInt8` + `Float32`, `UInt16` + `Int8`, and so on. Also, some variadic functions can accept any number of arguments, such as the `concat` function. + +Implementing a function may be slightly inconvenient because a function explicitly dispatches supported data types and supported `IColumns`. For example, the `plus` function has code generated by instantiation of a C++ template for each combination of numeric types, and for constant or non-constant left and right arguments. + +> This is a nice place to implement runtime code generation to avoid template code bloat. Also, it will make it possible to add fused functions like fused multiply-add, or to make multiple comparisons in one loop iteration. + +Due to vectorized query execution, functions are not short-circuit. For example, if you write `WHERE f(x) AND g(y)`, both sides will be calculated, even for rows, when `f(x)` is zero (except when `f(x)` is a zero constant expression). But if selectivity of the `f(x)` condition is high, and calculation of `f(x)` is much cheaper than `g(y)`, it's better to implement multi-pass calculation: first calculate `f(x)`, then filter columns by the result, and then calculate `g(y)` only for smaller, filtered chunks of data. + +## Aggregate Functions + +Aggregate functions are stateful functions. They accumulate passed values into some state, and allow you to get results from that state. They are managed with the `IAggregateFunction` interface. States can be rather simple (the state for `AggregateFunctionCount` is just a single `UInt64` value) or quite complex (the state of `AggregateFunctionUniqCombined` is a combination of a linear array, a hash table and a `HyperLogLog` probabilistic data structure). + +To deal with multiple states while executing a high-cardinality `GROUP BY` query, states are allocated in `Arena` (a memory pool), or they could be allocated in any suitable piece of memory. States can have a non-trivial constructor and destructor: for example, complex aggregation states can allocate additional memory themselves. This requires some attention to creating and destroying states and properly passing their ownership, to keep track of who and when will destroy states. + +Aggregation states can be serialized and deserialized to pass over the network during distributed query execution or to write them on disk where there is not enough RAM. They can even be stored in a table with the `DataTypeAggregateFunction` to allow incremental aggregation of data. + +> The serialized data format for aggregate function states is not versioned right now. This is ok if aggregate states are only stored temporarily. But we have the `AggregatingMergeTree` table engine for incremental aggregation, and people are already using it in production. This is why we should add support for backward compatibility when changing the serialized format for any aggregate function in the future. + +## Server + +The server implements several different interfaces: + +- An HTTP interface for any foreign clients. +- A TCP interface for the native ClickHouse client and for cross-server communication during distributed query execution. +- An interface for transferring data for replication. + +Internally, it is just a basic multithreaded server without coroutines, fibers, etc. Since the server is not designed to process a high rate of simple queries but is intended to process a relatively low rate of complex queries, each of them can process a vast amount of data for analytics. + +The server initializes the `Context` class with the necessary environment for query execution: the list of available databases, users and access rights, settings, clusters, the process list, the query log, and so on. This environment is used by interpreters. + +We maintain full backward and forward compatibility for the server TCP protocol: old clients can talk to new servers and new clients can talk to old servers. But we don't want to maintain it eternally, and we are removing support for old versions after about one year. + +> For all external applications, we recommend using the HTTP interface because it is simple and easy to use. The TCP protocol is more tightly linked to internal data structures: it uses an internal format for passing blocks of data and it uses custom framing for compressed data. We haven't released a C library for that protocol because it requires linking most of the ClickHouse codebase, which is not practical. + +## Distributed query execution + +Servers in a cluster setup are mostly independent. You can create a `Distributed` table on one or all servers in a cluster. The `Distributed` table does not store data itself – it only provides a "view" to all local tables on multiple nodes of a cluster. When you SELECT from a `Distributed` table, it rewrites that query, chooses remote nodes according to load balancing settings, and sends the query to them. The `Distributed` table requests remote servers to process a query just up to a stage where intermediate results from different servers can be merged. Then it receives the intermediate results and merges them. The distributed table tries to distribute as much work as possible to remote servers, and does not send much intermediate data over the network. + +> Things become more complicated when you have subqueries in IN or JOIN clauses and each of them uses a `Distributed` table. We have different strategies for execution of these queries. + +There is no global query plan for distributed query execution. Each node has its own local query plan for its part of the job. We only have simple one-pass distributed query execution: we send queries for remote nodes and then merge the results. But this is not feasible for difficult queries with high cardinality GROUP BYs or with a large amount of temporary data for JOIN: in such cases, we need to "reshuffle" data between servers, which requires additional coordination. ClickHouse does not support that kind of query execution, and we need to work on it. + +## Merge Tree + +`MergeTree` is a family of storage engines that supports indexing by primary key. The primary key can be an arbitary tuple of columns or expressions. Data in a `MergeTree` table is stored in "parts". Each part stores data in the primary key order (data is ordered lexicographically by the primary key tuple). All the table columns are stored in separate `column.bin` files in these parts. The files consist of compressed blocks. Each block is usually from 64 KB to 1 MB of uncompressed data, depending on the average value size. The blocks consist of column values placed contiguously one after the other. Column values are in the same order for each column (the order is defined by the primary key), so when you iterate by many columns, you get values for the corresponding rows. + +The primary key itself is "sparse". It doesn't address each single row, but only some ranges of data. A separate `primary.idx` file has the value of the primary key for each N-th row, where N is called `index_granularity` (usually, N = 8192). Also, for each column, we have `column.mrk` files with "marks," which are offsets to each N-th row in the data file. Each mark is a pair: the offset in the file to the beginning of the compressed block, and the offset in the decompressed block to the beginning of data. Usually compressed blocks are aligned by marks, and the offset in the decompressed block is zero. Data for `primary.idx` always resides in memory and data for `column.mrk` files is cached. + +When we are going to read something from a part in `MergeTree`, we look at `primary.idx` data and locate ranges that could possibly contain requested data, then look at `column.mrk` data and calculate offsets for where to start reading those ranges. Because of sparseness, excess data may be read. ClickHouse is not suitable for a high load of simple point queries, because the entire range with `index_granularity` rows must be read for each key, and the entire compressed block must be decompressed for each column. We made the index sparse because we must be able to maintain trillions of rows per single server without noticeable memory consumption for the index. Also, because the primary key is sparse, it is not unique: it cannot check the existence of the key in the table at INSERT time. You could have many rows with the same key in a table. + +When you `INSERT` a bunch of data into `MergeTree`, that bunch is sorted by primary key order and forms a new part. To keep the number of parts relatively low, there are background threads that periodically select some parts and merge them to a single sorted part. That's why it is called `MergeTree`. Of course, merging leads to "write amplification". All parts are immutable: they are only created and deleted, but not modified. When SELECT is run, it holds a snapshot of the table (a set of parts). After merging, we also keep old parts for some time to make recovery after failure easier, so if we see that some merged part is probably broken, we can replace it with its source parts. + +`MergeTree` is not an LSM tree because it doesn't contain "memtable" and "log": inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. + +> MergeTree tables can only have one (primary) index: there aren't any secondary indices. It would be nice to allow multiple physical representations under one logical table, for example, to store data in more than one physical order or even to allow representations with pre-aggregated data along with original data. + +There are MergeTree engines that are doing additional work during background merges. Examples are `CollapsingMergeTree` and `AggregatingMergeTree`. This could be treated as special support for updates. Keep in mind that these are not real updates because users usually have no control over the time when background merges will be executed, and data in a `MergeTree` table is almost always stored in more than one part, not in completely merged form. + +## Replication + +Replication in ClickHouse is implemented on a per-table basis. You could have some replicated and some non-replicated tables on the same server. You could also have tables replicated in different ways, such as one table with two-factor replication and another with three-factor. + +Replication is implemented in the `ReplicatedMergeTree` storage engine. The path in `ZooKeeper` is specified as a parameter for the storage engine. All tables with the same path in `ZooKeeper` become replicas of each other: they synchronize their data and maintain consistency. Replicas can be added and removed dynamically simply by creating or dropping a table. + +Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse doesn't support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. + +Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop partition, etc. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the "get part" action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. To achieve this, one replica is elected as the leader, and that replica initiates merges and writes "merge parts" actions to the log. + +Replication is physical: only compressed parts are transferred between nodes, not queries. To lower the network cost (to avoid network amplification), merges are processed on each replica independently in most cases. Large merged parts are sent over the network only in cases of significant replication lag. + +In addition, each replica stores its state in ZooKeeper as the set of parts and its checksums. When the state on the local filesystem diverges from the reference state in ZooKeeper, the replica restores its consistency by downloading missing and broken parts from other replicas. When there is some unexpected or broken data in the local filesystem, ClickHouse does not remove it, but moves it to a separate directory and forgets it. + +> The ClickHouse cluster consists of independent shards, and each shard consists of replicas. The cluster is not elastic, so after adding a new shard, data is not rebalanced between shards automatically. Instead, the cluster load will be uneven. This implementation gives you more control, and it is fine for relatively small clusters such as tens of nodes. But for clusters with hundreds of nodes that we are using in production, this approach becomes a significant drawback. We should implement a table engine that will span its data across the cluster with dynamically replicated regions that could be split and balanced between clusters automatically. + diff --git a/docs/en/development/architecture.rst b/docs/en/development/architecture.rst deleted file mode 100644 index a49a8220d99..00000000000 --- a/docs/en/development/architecture.rst +++ /dev/null @@ -1,227 +0,0 @@ -Overview of ClickHouse architecture -=================================== - -ClickHouse is a true column-oriented DBMS. Data is stored by columns, and during query execution data is processed by arrays (vectors or chunks of columns). Whenever possible, operations are dispatched on arrays, rather than on individual values. This is called "vectorized query execution," and it helps lower dispatch cost relative to the cost of actual data processing. - - This idea is nothing new. It dates back to the ``APL`` programming language and its descendants: ``A+``, ``J``, ``K``, and ``Q``. Array programming is widely used in scientific data processing. Neither is this idea something new in relational databases: for example, it is used in the ``Vectorwise`` system. - - There are two different approaches for speeding up query processing: vectorized query execution and runtime code generation. In the latter, the code is generated for every kind of query on the fly, removing all indirection and dynamic dispatch. Neither of these approaches is strictly better than the other. Runtime code generation can be better when it fuses many operations together, thus fully utilizing CPU execution units and the pipeline. Vectorized query execution can be less practical, because it involves temporary vectors that must be written to cache and read back. If the temporary data does not fit in the L2 cache, this becomes an issue. But vectorized query execution more easily utilizes the SIMD capabilities of the CPU. A `research paper `_ written by our friends shows that it is better to combine both approaches. ClickHouse mainly uses vectorized query execution and has limited initial support for runtime code generation (only the inner loop of the first stage of GROUP BY can be compiled). - - -Columns -------- - -To represent columns in memory (actually, chunks of columns), the ``IColumn`` interface is used. This interface provides helper methods for implementation of various relational operators. Almost all operations are immutable: they do not modify the original column, but create a new modified one. For example, the ``IColumn::filter`` method accepts a filter byte mask and creates a new filtered column. It is used for the ``WHERE`` and ``HAVING`` relational operators. Additional examples: the ``IColumn::permute`` method to support ``ORDER BY``, the ``IColumn::cut`` method to support ``LIMIT``, and so on. - -Various ``IColumn`` implementations (``ColumnUInt8``, ``ColumnString`` and so on) are responsible for the memory layout of columns. Memory layout is usually a contiguous array. For the integer type of columns it is just one contiguous array, like ``std::vector``. For ``String`` and ``Array`` columns, it is two vectors: one for all array elements, placed contiguously, and a second one for offsets to the beginning of each array. There is also ``ColumnConst`` that stores just one value in memory, but looks like a column. - - -Field ------ - -Nevertheless, it is possible to work with individual values as well. To represent an individual value, ``Field`` is used. ``Field`` is just a discriminated union of ``UInt64``, ``Int64``, ``Float64``, ``String`` and ``Array``. ``IColumn`` has the ``operator[]`` method to get the n-th value as a ``Field``, and the ``insert`` method to append a ``Field`` to the end of a column. These methods are not very efficient, because they require dealing with temporary ``Field`` objects representing an individual value. There are more efficient methods, such as ``insertFrom``, ``insertRangeFrom``, and so on. - -``Field`` doesn't have enough information about a specific data type for a table. For example, ``UInt8``, ``UInt16``, ``UInt32``, and ``UInt64`` are all represented as ``UInt64`` in a ``Field``. - - -Leaky abstractions ------------------- - -``IColumn`` has methods for common relational transformations of data, but they don't meet all needs. For example, ``ColumnUInt64`` doesn't have a method to calculate the sum of two columns, and ``ColumnString`` doesn't have a method to run a substring search. These countless routines are implemented outside of ``IColumn``. - -Various functions on columns can be implemented in a generic, non-efficient way using ``IColumn`` methods to extract ``Field`` values, or in a specialized way using knowledge of inner memory layout of data in a specific ``IColumn`` implementation. To do this, functions are cast to a specific ``IColumn`` type and deal with internal representation directly. For example, ``ColumnUInt64`` has the ``getData`` method that returns a reference to an internal array, then a separate routine reads or fills that array directly. In fact, we have "leaky abstractions" to allow efficient specializations of various routines. - - -Data types ----------- - -``IDataType`` is responsible for serialization and deserialization: for reading and writing chunks of columns or individual values in binary or text form. -``IDataType`` directly corresponds to data types in tables. For example, there are ``DataTypeUInt32``, ``DataTypeDateTime``, ``DataTypeString`` and so on. - -``IDataType`` and ``IColumn`` are only loosely related to each other. Different data types can be represented in memory by the same ``IColumn`` implementations. For example, ``DataTypeUInt32`` and ``DataTypeDateTime`` are both represented by ``ColumnUInt32`` or ``ColumnConstUInt32``. In addition, the same data type can be represented by different ``IColumn`` implementations. For example, ``DataTypeUInt8`` can be represented by ``ColumnUInt8`` or ``ColumnConstUInt8``. - -``IDataType`` only stores metadata. For instance, ``DataTypeUInt8`` doesn't store anything at all (except vptr) and ``DataTypeFixedString`` stores just ``N`` (the size of fixed-size strings). - -``IDataType`` has helper methods for various data formats. Examples are methods to serialize a value with possible quoting, to serialize a value for JSON, and to serialize a value as part of XML format. There is no direct correspondence to data formats. For example, the different data formats ``Pretty`` and ``TabSeparated`` can use the same ``serializeTextEscaped`` helper method from the ``IDataType`` interface. - - -Block ------ - -A ``Block`` is a container that represents a subset (chunk) of a table in memory. It is just a set of triples: ``(IColumn, IDataType, column name)``. During query execution, data is processed by ``Block`` s. If we have a ``Block``, we have data (in the ``IColumn`` object), we have information about its type (in ``IDataType``) that tells us how to deal with that column, and we have the column name (either the original column name from the table, or some artificial name assigned for getting temporary results of calculations). - -When we calculate some function over columns in a block, we add another column with its result to the block, and we don't touch columns for arguments of the function because operations are immutable. Later, unneeded columns can be removed from the block, but not modified. This is convenient for elimination of common subexpressions. - -Blocks are created for every processed chunk of data. Note that for the same type of calculation, the column names and types remain the same for different blocks, and only column data changes. It is better to split block data from the block header, because small block sizes will have a high overhead of temporary strings for copying shared_ptrs and column names. - - -Block Streams -------------- - -Block streams are for processing data. We use streams of blocks to read data from somewhere, perform data transformations, or write data to somewhere. ``IBlockInputStream`` has the ``read`` method to fetch the next block while available. ``IBlockOutputStream`` has the ``write`` method to push the block somewhere. - -Streams are responsible for: - -#. Reading or writing to a table. The table just returns a stream for reading or writing blocks. -#. Implementing data formats. For example, if you want to output data to a terminal in ``Pretty`` format, you create a block output stream where you push blocks, and it formats them. -#. Performing data transformations. Let's say you have ``IBlockInputStream`` and want to create a filtered stream. You create ``FilterBlockInputStream`` and initialize it with your stream. Then when you pull a block from ``FilterBlockInputStream``, it pulls a block from your stream, filters it, and returns the filtered block to you. Query execution pipelines are represented this way. - -There are more sophisticated transformations. For example, when you pull from ``AggregatingBlockInputStream``, it reads all data from its source, aggregates it, and then returns a stream of aggregated data for you. Another example: ``UnionBlockInputStream`` accepts many input sources in the constructor and also a number of threads. It launches multiple threads and reads from multiple sources in parallel. - - Block streams use the "pull" approach to control flow: when you pull a block from the first stream, it consequently pulls the required blocks from nested streams, and the entire execution pipeline will work. Neither "pull" nor "push" is the best solution, because control flow is implicit, and that limits implementation of various features like simultaneous execution of multiple queries (merging many pipelines together). This limitation could be overcome with coroutines or just running extra threads that wait for each other. We may have more possibilities if we make control flow explicit: if we locate the logic for passing data from one calculation unit to another outside of those calculation units. Read this `nice article `_ for more thoughts. - - We should note that the query execution pipeline creates temporary data at each step. We try to keep block size small enough so that temporary data fits in the CPU cache. With that assumption, writing and reading temporary data is almost free in comparison with other calculations. We could consider an alternative, which is to fuse many operations in the pipeline together, to make the pipeline as short as possible and remove much of the temporary data. This could be an advantage, but it also has drawbacks. For example, a split pipeline makes it easy to implement caching intermediate data, stealing intermediate data from similar queries running at the same time, and merging pipelines for similar queries. - - -Formats -------- - -Data formats are implemented with block streams. There are "presentational" formats only suitable for output of data to the client, such as ``Pretty`` format, which provides only ``IBlockOutputStream``. And there are input/output formats, such as ``TabSeparated`` or ``JSONEachRow``. - -There are also row streams: ``IRowInputStream`` and ``IRowOutputStream``. They allow you to pull/push data by individual rows, not by blocks. And they are only needed to simplify implementation of row-oriented formats. The wrappers ``BlockInputStreamFromRowInputStream`` and ``BlockOutputStreamFromRowOutputStream`` allow you to convert row-oriented streams to regular block-oriented streams. - - -I/O ---- - -For byte-oriented input/output, there are ``ReadBuffer`` and ``WriteBuffer`` abstract classes. They are used instead of C++ ``iostream``'s. Don't worry: every mature C++ project is using something other than ``iostream``'s for good reasons. - -``ReadBuffer`` and ``WriteBuffer`` are just a contiguous buffer and a cursor pointing to the position in that buffer. Implementations may own or not own the memory for the buffer. There is a virtual method to fill the buffer with the following data (for ``ReadBuffer``) or to flush the buffer somewhere (for ``WriteBuffer``). The virtual methods are rarely called. - -Implementations of ``ReadBuffer``/``WriteBuffer`` are used for working with files and file descriptors and network sockets, for implementing compression (``CompressedWriteBuffer`` is initialized with another WriteBuffer and performs compression before writing data to it), and for other purposes – the names ``ConcatReadBuffer``, ``LimitReadBuffer``, and ``HashingWriteBuffer`` speak for themselves. - -Read/WriteBuffers only deal with bytes. To help with formatted input/output (for instance, to write a number in decimal format), there are functions from ``ReadHelpers`` and ``WriteHelpers`` header files. - -Let's look at what happens when you want to write a result set in ``JSON`` format to stdout. You have a result set ready to be fetched from ``IBlockInputStream``. You create ``WriteBufferFromFileDescriptor(STDOUT_FILENO)`` to write bytes to stdout. You create ``JSONRowOutputStream``, initialized with that ``WriteBuffer``, to write rows in ``JSON`` to stdout. You create ``BlockOutputStreamFromRowOutputStream`` on top of it, to represent it as ``IBlockOutputStream``. Then you call ``copyData`` to transfer data from ``IBlockInputStream`` to ``IBlockOutputStream``, and everything works. Internally, ``JSONRowOutputStream`` will write various JSON delimiters and call the ``IDataType::serializeTextJSON`` method with a reference to ``IColumn`` and the row number as arguments. Consequently, ``IDataType::serializeTextJSON`` will call a method from ``WriteHelpers.h``: for example, ``writeText`` for numeric types and ``writeJSONString`` for ``DataTypeString``. - - -Tables ------- - -Tables are represented by the ``IStorage`` interface. Different implementations of that interface are different table engines. Examples are ``StorageMergeTree``, ``StorageMemory``, and so on. Instances of these classes are just tables. - -The most important ``IStorage`` methods are ``read`` and ``write``. There are also ``alter``, ``rename``, ``drop``, and so on. The ``read`` method accepts the following arguments: the set of columns to read from a table, the ``AST`` query to consider, and the desired number of streams to return. It returns one or multiple ``IBlockInputStream`` objects and information about the stage of data processing that was completed inside a table engine during query execution. - -In most cases, the read method is only responsible for reading the specified columns from a table, not for any further data processing. All further data processing is done by the query interpreter and is outside the responsibility of ``IStorage``. - -But there are notable exceptions: -- The AST query is passed to the ``read`` method and the table engine can use it to derive index usage and to read less data from a table. -- Sometimes the table engine can process data itself to a specific stage. For example, ``StorageDistributed`` can send a query to remote servers, ask them to process data to a stage where data from different remote servers can be merged, and return that preprocessed data. -The query interpreter then finishes processing the data. - -The table's ``read`` method can return multiple ``IBlockInputStream`` objects to allow parallel data processing. These multiple block input streams can read from a table in parallel. Then you can wrap these streams with various transformations (such as expression evaluation or filtering) that can be calculated independently and create a ``UnionBlockInputStream`` on top of them, to read from multiple streams in parallel. - -There are also ``TableFunction`` s. These are functions that return a temporary ``IStorage`` object to use in the ``FROM`` clause of a query. - -To get a quick idea of how to implement your own table engine, look at something simple, like ``StorageMemory`` or ``StorageTinyLog``. - - As the result of the ``read`` method, ``IStorage`` returns ``QueryProcessingStage`` – information about what parts of the query were already calculated inside storage. Currently we have only very coarse granularity for that information. There is no way for the storage to say "I have already processed this part of the expression in WHERE, for this range of data". We need to work on that. - - -Parsers -------- - -A query is parsed by a hand-written recursive descent parser. For example, ``ParserSelectQuery`` just recursively calls the underlying parsers for various parts of the query. Parsers create an ``AST``. The ``AST`` is represented by nodes, which are instances of ``IAST``. - - Parser generators are not used for historical reasons. - - -Interpreters ------------- - -Interpreters are responsible for creating the query execution pipeline from an ``AST``. There are simple interpreters, such as ``InterpreterExistsQuery`` and ``InterpreterDropQuery``, or the more sophisticated ``InterpreterSelectQuery``. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the ``SELECT`` query is the ``IBlockInputStream`` to read the result set from; the result of the INSERT query is the ``IBlockOutputStream`` to write data for insertion to; and the result of interpreting the ``INSERT SELECT`` query is the ``IBlockInputStream`` that returns an empty result set on the first read, but that copies data from ``SELECT`` to ``INSERT`` at the same time. - -``InterpreterSelectQuery`` uses ``ExpressionAnalyzer`` and ``ExpressionActions`` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. ``ExpressionAnalyzer`` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations or query. - - -Functions ---------- - -There are ordinary functions and aggregate functions. For aggregate functions, see the next section. - -Ordinary functions don't change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for ``Block``'s of data to implement vectorized query execution. - -There are some miscellaneous functions, like ``blockSize``, ``rowNumberInBlock``, and ``runningAccumulate``, that exploit block processing and violate the independence of rows. - -ClickHouse has strong typing, so implicit type conversion doesn't occur. If a function doesn't support a specific combination of types, an exception will be thrown. But functions can work (be overloaded) for many different combinations of types. For example, the ``plus`` function (to implement the ``+`` operator) works for any combination of numeric types: ``UInt8`` + ``Float32``, ``UInt16`` + ``Int8``, and so on. Also, some variadic functions can accept any number of arguments, such as the ``concat`` function. - -Implementing a function may be slightly inconvenient because a function explicitly dispatches supported data types and supported ``IColumns``. For example, the ``plus`` function has code generated by instantiation of a C++ template for each combination of numeric types, and for constant or non-constant left and right arguments. - - This is a nice place to implement runtime code generation to avoid template code bloat. Also, it will make it possible to add fused functions like fused multiply-add, or to make multiple comparisons in one loop iteration. - - Due to vectorized query execution, functions are not short-circuit. For example, if you write ``WHERE f(x) AND g(y)``, both sides will be calculated, even for rows, when ``f(x)`` is zero (except when ``f(x)`` is a zero constant expression). But if selectivity of the ``f(x)`` condition is high, and calculation of ``f(x)`` is much cheaper than ``g(y)``, it's better to implement multi-pass calculation: first calculate ``f(x)``, then filter columns by the result, and then calculate ``g(y)`` only for smaller, filtered chunks of data. - - -Aggregate Functions -------------------- - -Aggregate functions are stateful functions. They accumulate passed values into some state, and allow you to get results from that state. They are managed with the ``IAggregateFunction`` interface. States can be rather simple (the state for ``AggregateFunctionCount`` is just a single ``UInt64`` value) or quite complex (the state of ``AggregateFunctionUniqCombined`` is a combination of a linear array, a hash table and a ``HyperLogLog`` probabilistic data structure). - -To deal with multiple states while executing a high-cardinality ``GROUP BY`` query, states are allocated in ``Arena`` (a memory pool), or they could be allocated in any suitable piece of memory. States can have a non-trivial constructor and destructor: for example, complex aggregation states can allocate additional memory themselves. This requires some attention to creating and destroying states and properly passing their ownership, to keep track of who and when will destroy states. - -Aggregation states can be serialized and deserialized to pass over the network during distributed query execution or to write them on disk where there is not enough RAM. They can even be stored in a table with the ``DataTypeAggregateFunction`` to allow incremental aggregation of data. - - The serialized data format for aggregate function states is not versioned right now. This is ok if aggregate states are only stored temporarily. But we have the ``AggregatingMergeTree`` table engine for incremental aggregation, and people are already using it in production. This is why we should add support for backward compatibility when changing the serialized format for any aggregate function in the future. - - -Server ------- - -The server implements several different interfaces: -- An HTTP interface for any foreign clients. -- A TCP interface for the native ClickHouse client and for cross-server communication during distributed query execution. -- An interface for transferring data for replication. - -Internally, it is just a basic multithreaded server without coroutines, fibers, etc. Since the server is not designed to process a high rate of simple queries but is intended to process a relatively low rate of complex queries, each of them can process a vast amount of data for analytics. - -The server initializes the ``Context`` class with the necessary environment for query execution: the list of available databases, users and access rights, settings, clusters, the process list, the query log, and so on. This environment is used by interpreters. - -We maintain full backward and forward compatibility for the server TCP protocol: old clients can talk to new servers and new clients can talk to old servers. But we don't want to maintain it eternally, and we are removing support for old versions after about one year. - - For all external applications, we recommend using the HTTP interface because it is simple and easy to use. The TCP protocol is more tightly linked to internal data structures: it uses an internal format for passing blocks of data and it uses custom framing for compressed data. We haven't released a C library for that protocol because it requires linking most of the ClickHouse codebase, which is not practical. - - -Distributed query execution ---------------------------- - -Servers in a cluster setup are mostly independent. You can create a ``Distributed`` table on one or all servers in a cluster. The ``Distributed`` table does not store data itself – it only provides a "view" to all local tables on multiple nodes of a cluster. When you SELECT from a ``Distributed`` table, it rewrites that query, chooses remote nodes according to load balancing settings, and sends the query to them. The ``Distributed`` table requests remote servers to process a query just up to a stage where intermediate results from different servers can be merged. Then it receives the intermediate results and merges them. The distributed table tries to distribute as much work as possible to remote servers, and does not send much intermediate data over the network. - - Things become more complicated when you have subqueries in IN or JOIN clauses and each of them uses a ``Distributed`` table. We have different strategies for execution of these queries. - - There is no global query plan for distributed query execution. Each node has its own local query plan for its part of the job. We only have simple one-pass distributed query execution: we send queries for remote nodes and then merge the results. But this is not feasible for difficult queries with high cardinality GROUP BYs or with a large amount of temporary data for JOIN: in such cases, we need to "reshuffle" data between servers, which requires additional coordination. ClickHouse does not support that kind of query execution, and we need to work on it. - - -Merge Tree ----------- - -``MergeTree`` is a family of storage engines that supports indexing by primary key. The primary key can be an arbitary tuple of columns or expressions. Data in a ``MergeTree`` table is stored in "parts". Each part stores data in the primary key order (data is ordered lexicographically by the primary key tuple). All the table columns are stored in separate ``column.bin`` files in these parts. The files consist of compressed blocks. Each block is usually from 64 KB to 1 MB of uncompressed data, depending on the average value size. The blocks consist of column values placed contiguously one after the other. Column values are in the same order for each column (the order is defined by the primary key), so when you iterate by many columns, you get values for the corresponding rows. - -The primary key itself is "sparse". It doesn't address each single row, but only some ranges of data. A separate ``primary.idx`` file has the value of the primary key for each N-th row, where N is called ``index_granularity`` (usually, N = 8192). Also, for each column, we have ``column.mrk`` files with "marks," which are offsets to each N-th row in the data file. Each mark is a pair: the offset in the file to the beginning of the compressed block, and the offset in the decompressed block to the beginning of data. Usually compressed blocks are aligned by marks, and the offset in the decompressed block is zero. Data for ``primary.idx`` always resides in memory and data for ``column.mrk`` files is cached. - -When we are going to read something from a part in ``MergeTree``, we look at ``primary.idx`` data and locate ranges that could possibly contain requested data, then look at ``column.mrk`` data and calculate offsets for where to start reading those ranges. Because of sparseness, excess data may be read. ClickHouse is not suitable for a high load of simple point queries, because the entire range with ``index_granularity`` rows must be read for each key, and the entire compressed block must be decompressed for each column. We made the index sparse because we must be able to maintain trillions of rows per single server without noticeable memory consumption for the index. Also, because the primary key is sparse, it is not unique: it cannot check the existence of the key in the table at INSERT time. You could have many rows with the same key in a table. - -When you ``INSERT`` a bunch of data into ``MergeTree``, that bunch is sorted by primary key order and forms a new part. To keep the number of parts relatively low, there are background threads that periodically select some parts and merge them to a single sorted part. That's why it is called ``MergeTree``. Of course, merging leads to "write amplification". All parts are immutable: they are only created and deleted, but not modified. When SELECT is run, it holds a snapshot of the table (a set of parts). After merging, we also keep old parts for some time to make recovery after failure easier, so if we see that some merged part is probably broken, we can replace it with its source parts. - -``MergeTree`` is not an LSM tree because it doesn't contain "memtable" and "log": inserted data is written directly to the filesystem. This makes it suitable only to INSERT data in batches, not by individual row and not very frequently – about once per second is ok, but a thousand times a second is not. We did it this way for simplicity's sake, and because we are already inserting data in batches in our applications. - - MergeTree tables can only have one (primary) index: there aren't any secondary indices. It would be nice to allow multiple physical representations under one logical table, for example, to store data in more than one physical order or even to allow representations with pre-aggregated data along with original data. - - There are MergeTree engines that are doing additional work during background merges. Examples are ``CollapsingMergeTree`` and ``AggregatingMergeTree``. This could be treated as special support for updates. Keep in mind that these are not real updates because users usually have no control over the time when background merges will be executed, and data in a ``MergeTree`` table is almost always stored in more than one part, not in completely merged form. - - -Replication ------------ - -Replication in ClickHouse is implemented on a per-table basis. You could have some replicated and some non-replicated tables on the same server. You could also have tables replicated in different ways, such as one table with two-factor replication and another with three-factor. - -Replication is implemented in the ``ReplicatedMergeTree`` storage engine. The path in ``ZooKeeper`` is specified as a parameter for the storage engine. All tables with the same path in ``ZooKeeper`` become replicas of each other: they synchronise their data and maintain consistency. Replicas can be added and removed dynamically simply by creating or dropping a table. - -Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with ``ZooKeeper``, and data is replicated to all other replicas asynchronously. Because ClickHouse doesn't support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. - -Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop partition, etc. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the "get part" action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. To achieve this, one replica is elected as the leader, and that replica initiates merges and writes "merge parts" actions to the log. - -Replication is physical: only compressed parts are transferred between nodes, not queries. To lower the network cost (to avoid network amplification), merges are processed on each replica independently in most cases. Large merged parts are sent over the network only in cases of significant replication lag. - -In addition, each replica stores its state in ZooKeeper as the set of parts and its checksums. When the state on the local filesystem diverges from the reference state in ZooKeeper, the replica restores its consistency by downloading missing and broken parts from other replicas. When there is some unexpected or broken data in the local filesystem, ClickHouse does not remove it, but moves it to a separate directory and forgets it. - - The ClickHouse cluster consists of independent shards, and each shard consists of replicas. The cluster is not elastic, so after adding a new shard, data is not rebalanced between shards automatically. Instead, the cluster load will be uneven. This implementation gives you more control, and it is fine for relatively small clusters such as tens of nodes. But for clusters with hundreds of nodes that we are using in production, this approach becomes a significant drawback. We should implement a table engine that will span its data across the cluster with dynamically replicated regions that could be split and balanced between clusters automatically. diff --git a/docs/en/development/build.md b/docs/en/development/build.md new file mode 100644 index 00000000000..fe8424c0e0d --- /dev/null +++ b/docs/en/development/build.md @@ -0,0 +1,128 @@ +# How to build ClickHouse on Linux + +Build should work on Linux Ubuntu 12.04, 14.04 or newer. +With appropriate changes, it should also work on any other Linux distribution. +The build process is not intended to work on Mac OS X. +Only x86_64 with SSE 4.2 is supported. Support for AArch64 is experimental. + +To test for SSE 4.2, do + +```bash +grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +## Install Git and CMake + +```bash +sudo apt-get install git cmake3 +``` + +Or just cmake on newer systems. + +## Detect the number of threads + +```bash +export THREADS=$(grep -c ^processor /proc/cpuinfo) +``` + +## Install GCC 7 + +There are several ways to do this. + +### Install from a PPA package + +```bash +sudo apt-get install software-properties-common +sudo apt-add-repository ppa:ubuntu-toolchain-r/test +sudo apt-get update +sudo apt-get install gcc-7 g++-7 +``` + +### Install from sources + +Look at [https://github.com/yandex/ClickHouse/blob/master/utils/prepare-environment/install-gcc.sh] + +## Use GCC 7 for builds + +```bash +export CC=gcc-7 +export CXX=g++-7 +``` + +## Install required libraries from packages + +```bash +sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev +``` + +## Checkout ClickHouse sources + +To get the latest stable version: + +```bash +git clone -b stable --recursive git@github.com:yandex/ClickHouse.git +# or: git clone -b stable --recursive https://github.com/yandex/ClickHouse.git + +cd ClickHouse +``` + +For development, switch to the `master` branch. +For the latest release candidate, switch to the `testing` branch. + +## Build ClickHouse + +There are two build variants. + +### Build release package + +Install prerequisites to build Debian packages. + +```bash +sudo apt-get install devscripts dupload fakeroot debhelper +``` + +Install the most recent version of Clang. + +Clang is embedded into the ClickHouse package and used at runtime. The minimum version is 5.0. It is optional. + +To install clang, see `utils/prepare-environment/install-clang.sh` + +You may also build ClickHouse with Clang for development purposes. +For production releases, GCC is used. + +Run the release script: + +```bash +rm -f ../clickhouse*.deb +./release +``` + +You will find built packages in the parent directory: + +```bash +ls -l ../clickhouse*.deb +``` + +Note that usage of debian packages is not required. +ClickHouse has no runtime dependencies except libc, so it could work on almost any Linux. + +Installing freshly built packages on a development server: + +```bash +sudo dpkg -i ../clickhouse*.deb +sudo service clickhouse-server start +``` + +### Build to work with code + +```bash +mkdir build +cd build +cmake .. +make -j $THREADS +cd .. +``` + +To create an executable, run `make clickhouse`. +This will create the `dbms/src/Server/clickhouse` executable, which can be used with `client` or `server` arguments. + diff --git a/docs/en/development/build.rst b/docs/en/development/build.rst deleted file mode 100644 index 800ba34239c..00000000000 --- a/docs/en/development/build.rst +++ /dev/null @@ -1,147 +0,0 @@ -How to build ClickHouse on Linux -================================ - -Build should work on Linux Ubuntu 12.04, 14.04 or newer. -With appropriate changes, build should work on any other Linux distribution. -Build is not intended to work on Mac OS X. -Only x86_64 with SSE 4.2 is supported. Support for AArch64 is experimental. - -To test for SSE 4.2, do - -.. code-block:: bash - - grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" - - -Install Git and CMake ---------------------- - -.. code-block:: bash - - sudo apt-get install git cmake3 - -Or just cmake on newer systems. - - -Detect number of threads ------------------------- - -.. code-block:: bash - - export THREADS=$(grep -c ^processor /proc/cpuinfo) - -Install GCC 7 -------------- - -There are several ways to do it. - -Install from PPA package -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - sudo apt-get install software-properties-common - sudo apt-add-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install gcc-7 g++-7 - - -Install from sources -~~~~~~~~~~~~~~~~~~~~ - -Look at [https://github.com/yandex/ClickHouse/blob/master/utils/prepare-environment/install-gcc.sh] - -Use GCC 7 for builds --------------------- - -.. code-block:: bash - - export CC=gcc-7 - export CXX=g++-7 - - -Install required libraries from packages ----------------------------------------- - -.. code-block:: bash - - sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev - - -Checkout ClickHouse sources ---------------------------- - -To get latest stable version: - -.. code-block:: bash - - git clone -b stable --recursive git@github.com:yandex/ClickHouse.git - # or: git clone -b stable --recursive https://github.com/yandex/ClickHouse.git - - cd ClickHouse - - -For development, switch to the ``master`` branch. -For latest release candidate, switch to the ``testing`` branch. - -Build ClickHouse ----------------- - -There are two variants of build. - -Build release package -~~~~~~~~~~~~~~~~~~~~~ - -Install prerequisites to build debian packages. - -.. code-block:: bash - - sudo apt-get install devscripts dupload fakeroot debhelper - -Install recent version of clang. - -Clang is embedded into ClickHouse package and used at runtime. Minimum version is 5.0. It is optional. - -To install clang, look at ``utils/prepare-environment/install-clang.sh`` - -You may also build ClickHouse with clang for development purposes. -For production releases, GCC is used. - -Run release script: - -.. code-block:: bash - - rm -f ../clickhouse*.deb - ./release - -You will find built packages in parent directory: - -.. code-block:: bash - - ls -l ../clickhouse*.deb - - -Note that usage of debian packages is not required. -ClickHouse has no runtime dependencies except libc, so it could work on almost any Linux. - -Installing just built packages on development server: - -.. code-block:: bash - - sudo dpkg -i ../clickhouse*.deb - sudo service clickhouse-server start - - -Build to work with code -~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - mkdir build - cd build - cmake .. - make -j $THREADS - cd .. - -To create an executable, run ``make clickhouse``. -This will create the ``dbms/src/Server/clickhouse`` executable, which can be used with ``client`` or ``server`` arguments. diff --git a/docs/en/development/build_osx.md b/docs/en/development/build_osx.md new file mode 100644 index 00000000000..a433d9e1a8a --- /dev/null +++ b/docs/en/development/build_osx.md @@ -0,0 +1,45 @@ +# How to build ClickHouse on Mac OS X + +Build should work on Mac OS X 10.12. If you're using earlier version, you can try to build ClickHouse using Gentoo Prefix and clang sl in this instruction. +With appropriate changes, it should also work on any other Linux distribution. + +## Install Homebrew + +```bash +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +``` + +## Install required compilers, tools, and libraries + +```bash +brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext homebrew/dupes/zlib readline boost --cc=gcc-7 +``` + +## Checkout ClickHouse sources + +To get the latest stable version: + +```bash +git clone -b stable --recursive --depth=10 git@github.com:yandex/ClickHouse.git +# or: git clone -b stable --recursive --depth=10 https://github.com/yandex/ClickHouse.git + +cd ClickHouse +``` + +For development, switch to the `master` branch. +For the latest release candidate, switch to the `testing` branch. + +## Build ClickHouse + +```bash +mkdir build +cd build +cmake .. -DCMAKE_CXX_COMPILER=`which g++-7` -DCMAKE_C_COMPILER=`which gcc-7` +make -j `sysctl -n hw.ncpu` +cd .. +``` + +## Caveats + +If you intend to run clickhouse-server, make sure to increase the system's maxfiles variable. See [MacOS.md](https://github.com/yandex/ClickHouse/blob/master/MacOS.md) for more details. + diff --git a/docs/en/development/build_osx.rst b/docs/en/development/build_osx.rst deleted file mode 100644 index 7ffbae9e110..00000000000 --- a/docs/en/development/build_osx.rst +++ /dev/null @@ -1,63 +0,0 @@ -How to build ClickHouse on Mac OS X -=================================== - -Build should work on Mac OS X 10.12. If you're using earlier version, you can try to build ClickHouse using Gentoo Prefix and clang sl in this instruction. -With appropriate changes, build should work on any other OS X distribution. - -Install Homebrew ----------------- - -.. code-block:: bash - - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - - -Install required compilers, tools, libraries --------------------------------------------- - -.. code-block:: bash - - brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext homebrew/dupes/zlib readline boost --cc=gcc-7 - - -Checkout ClickHouse sources ---------------------------- - -To get the latest stable version: - -.. code-block:: bash - - git clone -b stable --recursive --depth=10 git@github.com:yandex/ClickHouse.git - # or: git clone -b stable --recursive --depth=10 https://github.com/yandex/ClickHouse.git - - cd ClickHouse - -For development, switch to the ``master`` branch. -For the latest release candidate, switch to the ``testing`` branch. - -Build ClickHouse ----------------- - -.. code-block:: bash - - mkdir build - cd build - cmake .. -DCMAKE_CXX_COMPILER=`which g++-7` -DCMAKE_C_COMPILER=`which gcc-7` - make -j `sysctl -n hw.ncpu` - cd .. - -If you're using macOS 10.13 High Sierra, it's not possible to build it with GCC-7 due to `a bug `_. Build it with Clang 5.0 instead: - -.. code-block:: bash - - brew install llvm - mkdir build - cd build - export PATH="/usr/local/opt/llvm/bin:$PATH" - cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` -DLINKER_NAME=ld -DUSE_STATIC_LIBRARIES=OFF - make -j `sysctl -n hw.ncpu` clickhouse - -Caveats -------- - -If you intend to run clickhouse-server, make sure to increase system's maxfiles variable. See `MacOS.md `_ for more details. diff --git a/docs/en/development/index.md b/docs/en/development/index.md new file mode 100644 index 00000000000..44759f17870 --- /dev/null +++ b/docs/en/development/index.md @@ -0,0 +1,9 @@ +# ClickHouse Development + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/development/index.rst b/docs/en/development/index.rst deleted file mode 100644 index f5f961b500a..00000000000 --- a/docs/en/development/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -ClickHouse Development -====================== - -.. toctree:: - :glob: - - * diff --git a/docs/en/development/style.md b/docs/en/development/style.md new file mode 100644 index 00000000000..7f14247db15 --- /dev/null +++ b/docs/en/development/style.md @@ -0,0 +1,813 @@ +# How to write C++ code + +## General recommendations + +1. The following are recommendations, not requirements. +2. If you are editing code, it makes sense to follow the formatting of the existing code. +3. Code style is needed for consistency. Consistency makes it easier to read the code. and it also makes it easier to search the code. +4. Many of the rules do not have logical reasons; they are dictated by established practices. + +## Formatting + +1. Most of the formatting will be done automatically by `clang-format`. + +1. Offsets are 4 spaces. Configure your development environment so that a tab adds four spaces. + +1. A left curly bracket must be separated on a new line. (And the right one, as well.) + + ```cpp + inline void readBoolText(bool & x, ReadBuffer & buf) + { + char tmp = '0'; + readChar(tmp, buf); + x = tmp != '0'; + } + ``` + +1. But if the entire function body is quite short (a single statement), you can place it entirely on one line if you wish. Place spaces around curly braces (besides the space at the end of the line). + + ```cpp + inline size_t mask() const { return buf_size() - 1; } + inline size_t place(HashValue x) const { return x & mask(); } + ``` + +1. For functions, don't put spaces around brackets. + + ```cpp + void reinsert(const Value & x) + ``` + + ```cpp + memcpy(&buf[place_value], &x, sizeof(x)); + ``` + +1. When using statements such as if, for, and while (unlike function calls), put a space before the opening bracket. + + ```cpp + for (size_t i = 0; i < rows; i += storage.index_granularity) + ``` + +1. Put spaces around binary operators (+,-, *,/,%, ...), as well as the ternary operator?:. + + ```cpp + UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); + UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); + UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); + ``` + +1. If a line feed is entered, put the operator on a new line and increase the indent before it. + + ```cpp + if (elapsed_ns) + message << " (" + << rows_read_on_server * 1000000000 / elapsed_ns << " rows/s., " + << bytes_read_on_server * 1000.0 / elapsed_ns << " MB/s.) "; + ``` + +1. You can use spaces for alignment within a line, if desired. + + ```cpp + dst.ClickLogID = click.LogID; + dst.ClickEventID = click.EventID; + dst.ClickGoodEvent = click.GoodEvent; + ``` + +9. Don't use spaces around the operators `.`, `->` . + + If necessary, the operator can be wrapped to the next line. In this case, the offset in front of it is increased. + +10. Do not use a space to separate unary operators (`-, +, +, *, &`, ...) from the argument. + +11. Put a space after a comma, but not before it. The same rule goes for a semicolon inside a for expression. + +12. Do not use spaces to separate the `[]` operator. + +13. In a `template <...>` expression, use a space between `template`and`<`; no spaces after `<` or before `>`. + + ```cpp + template + struct AggregatedStatElement + {} + ``` + +14. In classes and structures, public, private, and protected are written on the same level as the class/struct, but all other internal elements should be deeper. + + ```cpp + template > + class MultiVersion + { + public: + /// The specific version of the object to use. + using Version = Ptr; + ... + } + ``` + +15. If the same namespace is used for the entire file, and there isn't anything else significant, an offset is not necessary inside namespace. + +16. If the block for if, for, while... expressions consists of a single statement, you don't need to use curly brackets. Place the statement on a separate line, instead. The same is true for a nested if, for, while... statement. But if the inner statement contains curly brackets or else, the external block should be written in curly brackets. + + ```cpp + /// Если файлы не открыты, то открываем их. + if (streams.empty()) + for (const auto & name : column_names) + streams.emplace(name, std::make_unique( + storage.files[name].data_file.path(), + storage.files[name].marks[mark_number].offset)); + ``` + +17. There should be any spaces at the ends of lines. + +18. Sources are UTF-8 encoded. + +19. Non-ASCII characters can be used in string literals. + + ```cpp + << ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; + ``` + +20. Do not write multiple expressions in a single line. + +21. Group sections of code inside functions and separate them with no more than one empty line. + +22. Separate functions, classes, and so on with at least one empty line (maximum – two empty lines). + +23. A const (related to a value) must be written before the type name. + + ``` + //correct + const char * pos + const std::string & s + //incorrect + char const * pos + ``` + +24. When declaring a pointer or reference, the \* and & symbols should be separated by spaces on both sides. + + ``` + //correct + const char * pos + //incorrect + const char* pos + const char *pos + ``` + +25. When using template types, alias them with the `using` keyword (except in the simplest cases). + + In other words, the template parameters are specified only in `using` and aren't repeated in the code. + + `using` can be declared locally, such as inside a function. + + ``` + //correct + using FileStreams = std::map>; + FileStreams streams; + //incorrect + std::map> streams; + ``` + +26. Do not declare several variables of different types in one statement. + + ``` + //incorrect + int x, *y; + ``` + +27. Do not use C-style casts. + + ```cpp + //incorrect + std::cerr << (int)c <<; std::endl; + //correct + std::cerr << static_cast(c) << std::endl; + ``` + +28. In classes and structs, group members and functions separately inside each visibility scope. + +29. For small classes and structs, it is not necessary to separate the method declaration from the implementation. + + The same is true for small methods in any classes or structs. + + For templated classes and structs, don't separate the method declarations from the implementation (because otherwise they must be defined in the same translation unit). + +30. You can wrap lines at 140 characters, instead of 80. + +31. Always use the prefix increment/decrement operators if postfix is not required. + + ```cpp + for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) + ``` + +## Comments + +1. Be sure to add comments for all non-trivial parts of code. + + This is very important. Writing the comment might help you realize that the code isn't necessary, or that it is designed wrong. + + ```cpp + /** How much of the piece of memory can be used. + * For example, if internal_buffer is 1 MB, and only 10 bytes were loaded to the buffer from the file for reading, + * then working_buffer will have a size of only 10 bytes + * (working_buffer.end() will point to the position right after those 10 bytes available for read). + */ + ``` + +2. Comments can be as detailed as necessary. + +3. Place comments before the code they describe. In rare cases, comments can come after the code, on the same line. + + ```cpp + /** Parses and executes the query. + */ + void executeQuery( + ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) + WriteBuffer & ostr, /// Where to write the result + Context & context, /// DB, tables, data types, engines, functions, aggregate functions... + BlockInputStreamPtr & query_plan, /// A description of query processing can be included here + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to + ) + ``` + +4. Comments should be written in English only. + +5. If you are writing a library, include detailed comments explaining it in the main header file. + +6. Do not add comments that do not provide additional information. In particular, do not leave empty comments like this: + + ```cpp + /* + * Procedure Name: + * Original procedure name: + * Author: + * Date of creation: + * Dates of modification: + * Modification authors: + * Original file name: + * Purpose: + * Intent: + * Designation: + * Classes used: + * Constants: + * Local variables: + * Parameters: + * Date of creation: + * Purpose: + */ + ``` + + (Example taken from: [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) + +7. Do not write garbage comments (author, creation date ..) at the beginning of each file. + +8. Single-line comments begin with three slashes: `///` and multi-line comments begin with `/**`. These comments are considered "documentation". + + Note: You can use Doxygen to generate documentation from these comments. But Doxygen is not generally used because it is more convenient to navigate the code in the IDE. + +9. Multi-line comments must not have empty lines at the beginning and end (except the line that closes a multi-line comment). + +10. For commenting out code, use basic comments, not “documenting” comments. + +1. Delete the commented out parts of the code before commiting. + +11. Do not use profanity in comments or code. + +12. Do not use uppercase letters. Do not use excessive punctuation. + + ```cpp + /// WHAT THE FAIL??? + ``` + +13. Do not make delimeters from comments. + + ``` + ///****************************************************** + ``` + +14. Do not start discussions in comments. + + ``` + /// Зачем ты сделал эту фигню? + ``` + +15. There's no need to write a comment at the end of a block describing what it was about. + + ``` + /// for + ``` + +## Names + +1. The names of variables and class members use lowercase letters with underscores. + + ```cpp + size_t max_block_size; + ``` + +2. The names of functions (methods) use camelCase beginning with a lowercase letter. + + ```cpp + std::string getName() const override { return "Memory"; } + ``` + +3. The names of classes (structures) use CamelCase beginning with an uppercase letter. Prefixes other than I are not used for interfaces. + + ```cpp + class StorageMemory : public IStorage + ``` + +4. The names of usings follow the same rules as classes, or you can add _t at the end. + +5. Names of template type arguments for simple cases: T; T, U; T1, T2. + + For more complex cases, either follow the rules for class names, or add the prefix T. + + ```cpp + template + struct AggregatedStatElement + ``` + +6. Names of template constant arguments: either follow the rules for variable names, or use N in simple cases. + + ```cpp + template + struct ExtractDomain + ``` + +7. For abstract classes (interfaces) you can add the I prefix. + + ```cpp + class IBlockInputStream + ``` + +8. If you use a variable locally, you can use the short name. + + In other cases, use a descriptive name that conveys the meaning. + + ```cpp + bool info_successfully_loaded = false; + ``` + +9. define‘s should be in ALL_CAPS with underscores. The same is true for global constants. + + ```cpp + #define MAX_SRC_TABLE_NAMES_TO_STORE 1000 + ``` + +10. File names should use the same style as their contents. + + If a file contains a single class, name the file the same way as the class, in CamelCase. + + If the file contains a single function, name the file the same way as the function, in camelCase. + +11. If the name contains an abbreviation, then: + - For variable names, the abbreviation should use lowercase letters `mysql_connection` (not `mySQL_connection`). + - For names of classes and functions, keep the uppercase letters in the abbreviation`MySQLConnection` (not `MySqlConnection`). + +12. Constructor arguments that are used just to initialize the class members should be named the same way as the class members, but with an underscore at the end. + + ```cpp + FileQueueProcessor( + const std::string & path_, + const std::string & prefix_, + std::shared_ptr handler_) + : path(path_), + prefix(prefix_), + handler(handler_), + log(&Logger::get("FileQueueProcessor")) + { + } + ``` + + The underscore suffix can be omitted if the argument is not used in the constructor body. + +13. There is no difference in the names of local variables and class members (no prefixes required). + + ``` + timer (не m_timer) + ``` + +14. Constants in enums use CamelCase beginning with an uppercase letter. ALL_CAPS is also allowed. If the enum is not local, use enum class. + + ```cpp + enum class CompressionMethod + { + QuickLZ = 0, + LZ4 = 1, + }; + ``` + +15. All names must be in English. Transliteration of Russian words is not allowed. + + ``` + not Stroka + ``` + +16. Abbreviations are acceptable if they are well known (when you can easily find the meaning of the abbreviation in Wikipedia or in a search engine). + + `AST`, `SQL`. + + Not `NVDH` (some random letters) + + Incomplete words are acceptable if the shortened version is common use. + + You can also use an abbreviation if the full name is included next to it in the comments. + +17. File names with C++ source code must have the .cpp extension. Header files must have the .h extension. + +## How to write code + +1. Memory management. + + Manual memory deallocation (delete) can only be used in library code. + + In library code, the delete operator can only be used in destructors. + + In application code, memory must be freed by the object that owns it. + + Examples: + - The easiest way is to place an object on the stack, or make it a member of another class. + - For a large number of small objects, use containers. + - For automatic deallocation of a small number of objects that reside in the heap, use shared_ptr/unique_ptr. + +2. Resource management. + + Use RAII and see the previous point. + +3. Error handling. + + Use exceptions. In most cases, you only need to throw an exception, and don't need to catch it (because of RAII). + + In offline data processing applications, it's often acceptable to not catch exceptions. + + In servers that handle user requests, it's usually enough to catch exceptions at the top level of the connection handler. + + In thread functions, you should catch and keep all exceptions to rethrow them in the main thread after join. + + ```cpp + /// If there were no other calculations yet, do it synchronously + if (!started) + { + calculate(); + started = true; + } + else /// If the calculations are already in progress, wait for results + pool.wait(); + + if (exception) + exception->rethrow(); + ``` + + Never hide exceptions without handling. Never just blindly put all exceptions to log. + + Not `catch (...) {}`. + + If you need to ignore some exceptions, do so only for specific ones and rethrow the rest. + + ```cpp + catch (const DB::Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION) + return nullptr; + else + throw; + } + ``` + + When using functions with response codes or errno, always check the result and throw an exception in case of error. + + ```cpp + if (0 != close(fd)) + throwFromErrno("Cannot close file " + file_name, ErrorCodes::CANNOT_CLOSE_FILE); + ``` + + Asserts are not used. + +4. Exception types. + + There is no need to use complex exception hierarchy in application code. The exception text should be understandable to a system administrator. + +5. Throwing exceptions from destructors. +This is not recommended, but it is allowed. + + Use the following options: + - Create a (done() or finalize()) function that will do all the work in advance that might lead to an exception. If that function was called, there should be no exceptions in the destructor later. + - Tasks that are too complex (such as sending messages over the network) can be put in separate method that the class user will have to call before destruction. + - If there is an exception in the destructor, it’s better to log it than to hide it (if the logger is available). + - In simple applications, it is acceptable to rely on std::terminate (for cases of noexcept by default in C++11) to handle exceptions. + +6. Anonymous code blocks. + + You can create a separate code block inside a single function in order to make certain variables local, so that the destructors are called when exiting the block. + + ```cpp + Block block = data.in->read();{ std::lock_guard lock(mutex); data.ready = true; data.block = block;}ready_any.set(); + ``` + +7. Multithreading. + + For offline data processing applications: + - Try to get the best possible performance on a single CPU core. You can then parallelize your code if necessary. + + In server applications: + - Use the thread pool to process requests. At this point, we haven't had any tasks that required userspace context switching. + + Fork is not used for parallelization. + +8. Synchronizing threads. + + Often it is possible to make different threads use different memory cells (even better: different cache lines,) and to not use any thread synchronization (except joinAll). + + If synchronization is required, in most cases, it is sufficient to use mutex under lock_guard. + + In other cases use system synchronization primitives. Do not use busy wait. + + Atomic operations should be used only in the simplest cases. + + Do not try to implement lock-free data structures unless it is your primary area of expertise. + +9. Pointers vs references. + + In most cases, prefer references. + +10. const. + + Use constant references, pointers to constants, const_iterator, const methods. + + Consider const to be default and use non-const only when necessary. + + When passing variable by value, using const usually does not make sense. + +11. unsigned. + + Use unsigned, if needed. + +12. Numeric types + + Use UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, and size_t, ssize_t, ptrdiff_t. + + Don't use signed/unsigned long, long long, short; signed char, unsigned char, or char types for numbers. + +13. Passing arguments. + + Pass complex values by reference (including std::string). + + If a function captures ownership of an objected created in the heap, make the argument type shared_ptr or unique_ptr. + +14. Returning values. + + In most cases, just use return. Do not write [return std::move(res)]{.strike}. + + If the function allocates an object on heap and returns it, use shared_ptr or unique_ptr. + + In rare cases you might need to return the value via an argument. In this case, the argument should be a reference. + + ```cpp + using AggregateFunctionPtr = std::shared_ptr; + + /** Creates an aggregate function by name. + */ + class AggregateFunctionFactory + { + public: + AggregateFunctionFactory(); + AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; + ``` + +15. namespace. + + There is no need to use a separate namespace for application code or small libraries. + + or small libraries. + + For medium to large libraries, put everything in the namespace. + + You can use the additional detail namespace in a library's .h file to hide implementation details. + + In a .cpp file, you can use the static or anonymous namespace to hide symbols. + + You can also use namespace for enums to prevent its names from polluting the outer namespace, but it’s better to use the enum class. + +16. Delayed initialization. + + If arguments are required for initialization then do not write a default constructor. + + If later you’ll need to delay initialization, you can add a default constructor that will create an invalid object. Or, for a small number of objects, you can use shared_ptr/unique_ptr. + + ```cpp + Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); + + /// For delayed initialization + Loader() {} + ``` + +17. Virtual functions. + + If the class is not intended for polymorphic use, you do not need to make functions virtual. This also applies to the destructor. + +18. Encodings. + + Use UTF-8 everywhere. Use `std::string`and`char *`. Do not use `std::wstring`and`wchar_t`. + +19. Logging. + + See the examples everywhere in the code. + + Before committing, delete all meaningless and debug logging, and any other types of debug output. + + Logging in cycles should be avoided, even on the Trace level. + + Logs must be readable at any logging level. + + Logging should only be used in application code, for the most part. + + Log messages must be written in English. + + The log should preferably be understandable for the system administrator. + + Do not use profanity in the log. + + Use UTF-8 encoding in the log. In rare cases you can use non-ASCII characters in the log. + +20. I/O. + + Don't use iostreams in internal cycles that are critical for application performance (and never use stringstream). + + Use the DB/IO library instead. + +21. Date and time. + + See the DateLUT library. + +22. include. + + Always use `#pragma once` instead of include guards. + +23. using. + + The using namespace is not used. + + It's fine if you are 'using' something specific, but make it local inside a class or function. + +24. Do not use trailing return type for functions unless necessary. + + [auto f() -> void;]{.strike} + +25. Do not declare and init variables like this: + + ```cpp + auto s = std::string{"Hello"}; + ``` + + Do it like this: + + ```cpp + std::string s = "Hello"; + std::string s{"Hello"}; + ``` + +26. For virtual functions, write 'virtual' in the base class, but write 'override' in descendent classes. + +## Unused features of C++ + +1. Virtual inheritance is not used. +2. Exception specifiers from C++03 are not used. +3. Function try block is not used, except for the main function in tests. + +## Platform + +1. We write code for a specific platform. + + But other things being equal, cross-platform or portable code is preferred. + +2. The language is C++17. + +3. The compiler is gcc. At this time (December 2017), the code is compiled using version 7.2. (It can also be compiled using clang 5.) + + The standard library is used (implementation of libstdc++ or libc++). + +4. OS: Linux Ubuntu, not older than Precise. + +5. Code is written for x86_64 CPU architecture. + + The CPU instruction set is the minimum supported set among our servers. Currently, it is SSE 4.2. + +6. Use `-Wall -Werror` compilation flags. + +7. Use static linking with all libraries except those that are difficult to connect to statically (see the output of the 'ldd' command). + +8. Code is developed and debugged with release settings. + +## Tools + +1. KDevelop is a good IDE. + +2. For debugging, use gdb, valgrind (memcheck), strace,-fsanitize=..., tcmalloc_minimal_debug. + +3. For profiling, use Linux Perf valgrind (callgrind), strace-cf. + +4. Sources are in Git. + +5. Compilation is managed by CMake. + +6. Releases are in deb packages. + +7. Commits to master must not break the build. + + Though only selected revisions are considered workable. + +8. Make commits as often as possible, even if the code is only partially ready. + + Use branches for this purpose. + + If your code is not buildable yet, exclude it from the build before pushing to master. You'll need to finish it or remove it from master within a few days. + +9. For non-trivial changes, used branches and publish them on the server. + +10. Unused code is removed from the repository. + +## Libraries + +1. The C++14 standard library is used (experimental extensions are fine), as well as boost and Poco frameworks. + +2. If necessary, you can use any well-known libraries available in the OS package. + + If there is a good solution already available, then use it, even if it means you have to install another library. + + (But be prepared to remove bad libraries from code.) + +3. You can install a library that isn't in the packages, if the packages don't have what you need or have an outdated version or the wrong type of compilation. + +4. If the library is small and doesn't have its own complex build system, put the source files in the contrib folder. + +5. Preference is always given to libraries that are already used. + +## General recommendations + +1. Write as little code as possible. +2. Try the simplest solution. +3. Don't write code until you know how it's going to work and how the inner loop will function. +4. In the simplest cases, use 'using' instead of classes or structs. +5. If possible, do not write copy constructors, assignment operators, destructors (other than a virtual one, if the class contains at least one virtual function), mpve-constructors and move assignment operators. In other words, the compiler-generated functions must work correctly. You can use 'default'. +6. Code simplification is encouraged. Reduce the size of your code where possible. + +## Additional recommendations + +1. Explicit std:: for types from stddef.h + + is not recommended. We recommend writing size_t instead std::size_t because it's shorter. + + But if you prefer, std:: is acceptable. + +2. Explicit std:: for functions from the standard C library. + + is not recommended. Write memcpy instead of std::memcpy. + + The reason is that there are similar non-standard functions, such as memmem. We do use these functions on occasion. These functions do not exist in namespace std. + + If you write std::memcpy instead of memcpy everywhere, then memmem without std:: will look awkward. + + Nevertheless, std:: is allowed if you prefer it. + +3. Using functions from C when the ones are available in the standard C++ library. + + This is acceptable if it is more efficient. + + For example, use memcpy instead of std::copy for copying large chunks of memory. + +4. Multiline function arguments. + + Any of the following wrapping styles are allowed: + + ```cpp + function( + T1 x1, + T2 x2) + ``` + + ```cpp + function( + size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) + ``` + + ```cpp + function(size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) + ``` + + ```cpp + function(size_t left, size_t right, + const & RangesInDataParts ranges, + size_t limit) + ``` + + ```cpp + function( + size_t left, + size_t right, + const & RangesInDataParts ranges, + size_t limit) + ``` + diff --git a/docs/en/development/style.rst b/docs/en/development/style.rst deleted file mode 100644 index 41ad3813a72..00000000000 --- a/docs/en/development/style.rst +++ /dev/null @@ -1,731 +0,0 @@ -.. role:: strike - :class: strike - -How to write C++ code -===================== - -General -------- - -#. This text should be considered as recommendations. -#. If you edit some code, it makes sense to keep style in changes consistent with the rest of it. -#. Style is needed to keep code consistent. Consistency is required to make it easier (more convenient) to read the code. And also for code navigation. -#. Many rules do not have any logical explanation and just come from existing practice. - -Formatting ----------- - -#. Most of formatting is done automatically by ``clang-format``. -#. Indent is 4 spaces wide. Configure your IDE to insert 4 spaces on pressing Tab. -#. Curly braces on separate lines. - - .. code-block:: cpp - - inline void readBoolText(bool & x, ReadBuffer & buf) - { - char tmp = '0'; - readChar(tmp, buf); - x = tmp != '0'; - } - - -#. But if function body is short enough (one statement) you can put the whole thing on one line. In this case put spaces near curly braces, except for the last one in the end. - - .. code-block:: cpp - - inline size_t mask() const { return buf_size() - 1; } - inline size_t place(HashValue x) const { return x & mask(); } - - -#. For functions there are no spaces near brackets. - - .. code-block:: cpp - - void reinsert(const Value & x) - - .. code-block:: cpp - - memcpy(&buf[place_value], &x, sizeof(x)); - - -#. When using expressions if, for, while, ... (in contrast to function calls) there should be space before opening bracket. - - .. code-block:: cpp - - for (size_t i = 0; i < rows; i += storage.index_granularity) - -#. There should be spaces around binary operators (+, -, \*, /, %, ...) and ternary operator ?:. - - .. code-block:: cpp - - UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); - UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); - UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - - -#. If there's a line break, operator is written on new line and it has additional indent. - - .. code-block:: cpp - - if (elapsed_ns) - message << " (" - << rows_read_on_server * 1000000000 / elapsed_ns << " rows/s., " - << bytes_read_on_server * 1000.0 / elapsed_ns << " MB/s.) "; - - #. It is ok to insert additional spaces to align the code. - - .. code-block:: cpp - - dst.ClickLogID = click.LogID; - dst.ClickEventID = click.EventID; - dst.ClickGoodEvent = click.GoodEvent; - - -#. No spaces around ``.``, ``->`` operators. - If necessary these operators can be moved to next line with additional indent. - -#. Unary operators (``--, ++, *, &``, ...) are not delimited from argument. - -#. Space is put after comma or semicolon, not before. - -#. Operator ``[]`` is not delimited with spaces. - -#. In ``template <...>``, put space between ``template`` and ``<``; after ``<`` and before ``>`` - do not. - - .. code-block:: cpp - - template - struct AggregatedStatElement - - -#. In classes and structs keywords public, private, protected are written on same indention level as class/struct, while other contents - deeper. - - .. code-block:: cpp - - template - class MultiVersion - { - public: - /// Version of object for usage. shared_ptr manage lifetime of version. - using Version = std::shared_ptr; - - -#. If there's only one namespace in a file and there's nothing else significant - no need to indent the namespace. - -#. If ``if, for, while...`` block consists of only one statement, it's not required to wrap it in curly braces. Instead you can put the statement on separate line. This statements can also be a ``if, for, while...`` block. But if internal statement contains curly braces or else, this option should not be used. - - .. code-block:: cpp - - /// Finish write. - for (auto & stream : streams) - stream.second->finalize(); - -#. No spaces before end of line. - -#. Source code should be in UTF-8 encoding. - -#. It's ok to have non-ASCII characters in string literals. - - .. code-block:: cpp - - << ", " << (timer.elapsed() / chunks_stats.hits) << " μsec/hit."; - - -#. Don't put multiple statements on single line. - -#. Inside functions do not delimit logical blocks by more than one empty line. - -#. Functions, classes and similar constructs are delimited by one or two empty lines. - -#. const (related to value) is written before type name. - - .. code-block:: cpp - - const char * pos - - .. code-block:: cpp - - const std::string & s - - :strike:`char const * pos` - -#. When declaring pointer or reference symbols \* and & should be surrounded by spaces. - - .. code-block:: cpp - - const char * pos - - :strike:`const char\* pos` - :strike:`const char \*pos` - -#. Alias template types with ``using`` keyword (except the most simple cases). It can be declared even locally, for example inside functions. - - .. code-block:: cpp - - using FileStreams = std::map>; - FileStreams streams; - - :strike:`std::map> streams;` - -#. Do not declare several variables of different types in one statements. - - :strike:`int x, *y;` - -#. C-style casts should be avoided. - - :strike:`std::cerr << (int)c << std::endl;` - - .. code-block:: cpp - - std::cerr << static_cast(c) << std::endl; - - -#. In classes and structs group members and functions separately inside each visibility scope. - -#. For small classes and structs, it is not necessary to split method declaration and implementation. - The same for small methods. - For templated classes and structs it is better not to split declaration and implementations (because anyway they should be defined in the same translation unit). - -#. Lines should be wrapped at 140 symbols, not 80. - -#. Always use prefix increment/decrement if postfix is not required. - - .. code-block:: cpp - - for (Names::const_iterator it = column_names.begin(); it != column_names.end(); ++it) - - -Comments --------- - -#. You shoud write comments in all not trivial places. - It is very important. While writing comment you could even understand that code does the wrong thing or is completely unnecessary. - - .. code-block:: cpp - - /** Part of piece of memory, that can be used. - * For example, if internal_buffer is 1MB, and there was only 10 bytes loaded to buffer from file for reading, - * then working_buffer will have size of only 10 bytes - * (working_buffer.end() will point to position right after those 10 bytes available for read). - */ - - -#. Comments can be as detailed as necessary. - -#. Comments are written before the relevant code. In rare cases - after on the same line. - - .. code-block:: text - - /** Parses and executes the query. - */ - void executeQuery( - ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) - WriteBuffer & ostr, /// Where to write the result - Context & context, /// DB, tables, data types, engines, functions, aggregate functions... - BlockInputStreamPtr & query_plan, /// Here could be written the description on how query was executed - QueryProcessingStage::Enum stage = QueryProcessingStage::Complete); /// Up to which stage process the SELECT query - -#. Comments should be written only in english - -#. When writing a library, put it's detailed description in it's main header file. - -#. You shouldn't write comments not providing additional information. For instance, you *CAN'T* write empty comments like this one: - - .. code-block:: cpp - - /* - * Procedure Name: - * Original procedure name: - * Author: - * Date of creation: - * Dates of modification: - * Modification authors: - * Original file name: - * Purpose: - * Intent: - * Designation: - * Classes used: - * Constants: - * Local variables: - * Parameters: - * Date of creation: - * Purpose: - */ - - (example is borrowed from here: http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) - -#. You shouldn't write garbage comments (author, creation date...) in the beginning of each file. - -#. One line comments should start with three slashes: ``///``, multiline - with ``/**``. This comments are considered "documenting". - Note: such comments could be used to generate docs using Doxygen. But in reality Doxygen is not used because it is way more convenient to use IDE for code navigation. - -#. In beginning and end of multiline comments there should be no empty lines (except the one where the comment ends). - -#. For commented out code use simple, not "documenting" comments. Delete commented out code before commits. - -#. Do not use profanity in comments or code. - -#. Do not use too many question signs, exclamation points or capital letters. - :strike:`/// WHAT THE FAIL???` - -#. Do not make delimeters from comments. - :strike:`/*******************************************************/` - -#. Do not create discussions in comments. - :strike:`/// Why you did this?` - -#. Do not comment end of block describing what kind of block it was. - :strike:`} /// for` - - -Names ------ - -#. Names of variables and class members — in lowercase with underscores. - - .. code-block:: cpp - - size_t max_block_size; - -#. Names of functions (methids) - in camelCase starting with lowercase letter. - - .. code-block:: cpp - - std::string getName() const override { return "Memory"; } - -#. Names of classes (structs) - CamelCase starting with uppercase letter. Prefixes are not used, except I for interfaces. - - .. code-block:: cpp - - class StorageMemory : public IStorage - - -#. Names of ``using``'s - same as classes and can have _t suffix. - -#. Names of template type arguments: in simple cases - T; T, U; T1, T2. - In complex cases - like class names or can have T prefix. - - .. code-block:: cpp - - template - struct AggregatedStatElement - -#. Names of template constant arguments: same as variable names or N in simple cases. - - .. code-block:: cpp - - template - struct ExtractDomain - -#. For abstract classes (interfaces) you can add I to the start of name. - - .. code-block:: cpp - - class IBlockInputStream - -#. If variable is used pretty locally, you can use short name. - In other cases - use descriptive name. - - .. code-block:: cpp - - bool info_successfully_loaded = false; - - -#. ``define``'s should be in ALL_CAPS with underlines. Global constants - too. - - .. code-block:: cpp - - #define MAX_SRC_TABLE_NAMES_TO_STORE 1000 - -#. Names of files should match it's contents. - If file contains one class - name it like class in CamelCase. - If file contains one function - name it like function in camelCase. - -#. If name contains an abbreviation: - * for variables names it should be all lowercase; - ``mysql_connection`` - :strike:`mySQL_connection` - - * for class and function names it should be all uppercase; - ``MySQLConnection`` - :strike:`MySqlConnection` - -#. Constructor arguments used just to initialize the class members, should have the matching name, but with underscore suffix. - - .. code-block:: cpp - - FileQueueProcessor( - const std::string & path_, - const std::string & prefix_, - std::shared_ptr handler_) - : path(path_), - prefix(prefix_), - handler(handler_), - log(&Logger::get("FileQueueProcessor")) - { - } - - The underscore suffix can be omitted if argument is not used in constructor body. - -#. Naming of local variables and class members do not have any differences (no prefixes required). - ``timer`` - :strike:`m_timer` - -#. Constants in enums - CamelCase starting with uppercase letter. ALL_CAPS is also ok. If enum is not local, use enum class. - - .. code-block:: cpp - - enum class CompressionMethod - { - QuickLZ = 0, - LZ4 = 1, - }; - -#. All names - in English. Transliteration from Russian is not allowed. - :strike:`Stroka` - -#. Abbreviations are fine only if they are well known (when you can find what it means in wikipedia or with web search query). - - ``AST`` ``SQL`` - :strike:`NVDH (some random letters)` - - Using incomplete words is ok if it is commonly used. Also you can put the whole word in comments. - -#. C++ source code extensions should be .cpp. Header files - only .h. - :strike:`.hpp` :strike:`.cc` :strike:`.C` :strike:`.inl` - ``.inl.h`` is ok, but not :strike:`.h.inl:strike:` - - -How to write code ------------------ - -#. Memory management. - Manual memory deallocation (delete) is ok only in destructors in library code. - In application code memory should be freed by some object that owns it. - Examples: - * you can put object on stack or make it a member of another class. - * use containers for many small objects. - * for automatic deallocation of not that many objects residing in heap, use shared_ptr/unique_ptr. - -#. Resource management. - Use RAII and see abovee. - -#. Error handling. - Use exceptions. In most cases you should only throw exception, but not catch (because of RAII). - In offline data processing applications it's often ok not to catch exceptions. - In server code serving user requests usually you should catch exceptions only on top level of connection handler. - In thread functions you should catch and keep all exceptions to rethrow it in main thread after join. - - .. code-block:: cpp - - /// If there were no other calculations yet - lets do it synchronously - if (!started) - { - calculate(); - started = true; - } - else /// If the calculations are already in progress - lets wait - pool.wait(); - - if (exception) - exception->rethrow(); - - Never hide exceptions without handling. Never just blindly put all exceptions to log. - :strike:`catch (...) {}` - If you need to ignore some exceptions, do so only for specific ones and rethrow the rest.. - - .. code-block:: cpp - - catch (const DB::Exception & e) - { - if (e.code() == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION) - return nullptr; - else - throw; - } - - When using functions with error codes - always check it and throw exception in case of error. - - .. code-block:: cpp - - if (0 != close(fd)) - throwFromErrno("Cannot close file " + file_name, ErrorCodes::CANNOT_CLOSE_FILE); - - Asserts are not used. - -#. Exception types. - No need to use complex exception hierarchy in application code. Exception code should be understandable by operations engineer. - -#. Throwing exception from destructors. - Not recommended, but allowed. - Use the following options: - * Create function (done() or finalize()) that will in advance do all the work that might lead to exception. If that function was called, later there should be no exceptions in destructor. - * Too complex work (for example, sending messages via network) can be put in separate method that class user will have to call before destruction. - * If nevertheless there's an exception in destructor it's better to log it that to hide it. - * In simple applications it is ok to rely on std::terminate (in case of noexcept by default in C++11) to handle exception. - -#. Anonymous code blocks. - It is ok to declare anonymous code block to make some variables local to it and make them be destroyed earlier than they otherwise would. - - .. code-block:: cpp - - Block block = data.in->read(); - - { - std::lock_guard lock(mutex); - data.ready = true; - data.block = block; - } - - ready_any.set(); - -#. Multithreading. - In case of offline data processing applications: - * Try to make code as fast as possible on single core. - * Make it parallel only if single core performance appeared to be not enough. - In server application: - * use thread pool for request handling; - * for now there were no tasks where userspace context switching was really necessary. - Fork is not used to parallelize code. - -#. Synchronizing threads. - Often it is possible to make different threads use different memory cells (better - different cache lines) and do not use any synchronization (except joinAll). - If synchronization is necessary in most cases mutex under lock_guard is enough. - In other cases use system synchronization primitives. Do not use busy wait. - Atomic operations should be used only in the most simple cases. - Do not try to implement lock-free data structures unless it is your primary area of expertise. - -#. Pointers vs reference. - Prefer references. - -#. const. - Use constant references, pointers to constants, const_iterator, const methods. - Consider const to be default and use non-const only when necessary. - When passing variable by value using const usually do not make sense. - -#. unsigned. - unsinged is ok if necessary. - -#. Numeric types. - Use types UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, as well as size_t, ssize_t, ptrdiff_t. - Do not use signed/unsigned long, long long, short; signed char, unsigned char, аnd char. - -#. Passing arguments. - Pass complex values by reference (including std::string). - If functions captures object ownership created in heap, make an argument to be shared_ptr or unique_ptr. - -#. Returning values. - In most cases just use return. Do not write :strike:`return std::move(res)`. - If function allocates an object on heap and returns it, use shared_ptr or unique_ptr. - In rare cases you might need to return value via argument, in this cases the argument should be a reference. - - .. code-block:: cpp - - using AggregateFunctionPtr = std::shared_ptr; - - /** Creates aggregate function by it's name - */ - class AggregateFunctionFactory - { - public: - AggregateFunctionFactory(); - AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; - -#. namespace. - No need to use separate namespace for application code or small libraries. - For medium to large libraries - put everything in namespace. - You can use additional detail namespace in .h file to hide implementation details. - In .cpp you can use static or anonymous namespace to hide symbols. - You can also use namespace for enums to prevent it's names to pollute outer namespace, but it's better to use enum class. - -#. Delayed initialization. - If arguments are required for initialization then do not write default constructor. - If later you'll need to delay initialization you can add default constructor creating invalid object. - For small number of object you could use shared_ptr/unique_ptr. - - .. code-block:: cpp - - Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); - - /// For delayed initialization - Loader() {} - -#. Virtual methods. - Do not mark methods or destructor as virtual if class is not intended for polymorph usage. - -#. Encoding. - Always use UTF-8. Use ``std::string``, ``char *``. Do not use ``std::wstring``, ``wchar_t``. - -#. Logging. - See examples in code. - Remove debug logging before commit. - Logging in cycles should be avoided, even on Trace level. - Logs should be readable even with most detailed settings. - Log mostly in application code. - Log messages should be in English and understandable by operations engineers. - -#. I/O. - In internal cycle (critical for application performance) you can't use iostreams (especially stringstream). - Instead use DB/IO library. - -#. Date and time. - See DateLUT library. - -#. include. - Always use ``#pragma once`` instead of include guards. - -#. using. - Don't use ``using namespace``. ``using`` something specific is fine, try to put it as locally as possible. - -#. Do not use trailing return unless necessary. - :strike:`auto f() -> void;` - -#. Do not delcare and init variables like this: - :strike:`auto s = std::string{"Hello"};` - Do it like this instead: - ``std::string s = "Hello";`` - ``std::string s{"Hello"};`` - -#. For virtual functions write virtual in base class and don't forget to write override in descendant classes. - - -Unused C++ features -------------------- - -#. Virtual inheritance. - -#. Exception specifiers from C++03. - -#. Function try block, except main function in tests. - - -Platform --------- - -#. We write code for specific platform. But other things equal cross platform and portable code is preferred. - -#. Language is C++17. - -#. Compiler is gcc. As of December 2017 version 7.2 is used. It is also compilable with clang 5. - Standard library is used (libstdc++ or libc++). - -#. OS - Linux Ubuntu, not older than Precise. - -#. Code is written for x86_64 CPU architecture. - CPU instruction set is SSE4.2 is currently required. - -#. ``-Wall -Werror`` compilation flags are used. - -#. Static linking is used by default. See ldd output for list of exceptions from this rule. - -#. Code is developed and debugged with release settings. - - -Tools ------ - -#. Good IDE - KDevelop. - -#. For debugging gdb, valgrind (memcheck), strace, -fsanitize=..., tcmalloc_minimal_debug are used. - -#. For profiling - Linux Perf, valgrind (callgrind), strace -cf. - -#. Source code is in Git. - -#. Compilation is managed by CMake. - -#. Releases are in deb packages. - -#. Commits to master should not break build. - Though only selected revisions are considered workable. - -#. Commit as often as possible, even if code is not quite ready yet. - Use branches for this. - If your code is not buildable yet, exclude it from build before pushing to master; - you'll have few days to fix or delete it from master after that. - -#. For non-trivial changes use branches and publish them on server. - -#. Unused code is removed from repository. - - -Libraries ---------- - -#. C++14 standard library is used (experimental extensions are fine), as well as boost and Poco frameworks. - -#. If necessary you can use any well known library available in OS with packages. - If there's a good ready solution it is used even if it requires to install one more dependency. - (Buy be prepared to remove bad libraries from code.) - -#. It is ok to use library not from packages if it is not available or too old. - -#. If the library is small and does not have it's own complex build system, you should put it's sources in contrib folder. - -#. Already used libraries are preferred. - - -General -------- - -#. Write as short code as possible. - -#. Try the most simple solution. - -#. Do not write code if you do not know how it will work yet. - -#. In the most simple cases prefer using over classes and structs. - -#. Write copy constructors, assignment operators, destructor (except virtual), mpve-constructor and move assignment operators only if there's no other option. You can use ``default``. - -#. It is encouraged to simplify code. - - -Additional ----------- - -#. Explicit std:: for types from stddef.h. - Not recommended, but allowed. - -#. Explicit std:: for functions from C standard library. - Not recommended. For example, write memcpy instead of std::memcpy. - Sometimes there are non standard functions with similar names, like memmem. It will look weird to have memcpy with std:: prefix near memmem without. Though specifying std:: is not prohibited. - -#. Usage of C functions when there are alternatives in C++ standard library. - Allowed if they are more effective. For example, use memcpy instead of std::copy for copying large chunks of memory. - -#. Multiline function arguments. - All of the following styles are allowed: - - .. code-block:: cpp - - function( - T1 x1, - T2 x2) - - .. code-block:: cpp - - function( - size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) - - .. code-block:: cpp - - function(size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) - - .. code-block:: cpp - - function(size_t left, size_t right, - const & RangesInDataParts ranges, - size_t limit) - - .. code-block:: cpp - - function( - size_t left, - size_t right, - const & RangesInDataParts ranges, - size_t limit) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md new file mode 100644 index 00000000000..40ae8705424 --- /dev/null +++ b/docs/en/development/tests.md @@ -0,0 +1,32 @@ +# How to run ClickHouse tests + +The `clickhouse-test` utility that is used for functional testing is written using Python 2.x.It also requires you to have some third-party packages: + +```bash +$ pip install lxml termcolor +``` + +In a nutshell: + +- Put the `clickhouse` program to `/usr/bin` (or `PATH`) +- Create a `clickhouse-client` symlink in `/usr/bin` pointing to `clickhouse` +- Start the `clickhouse` server +- `cd dbms/tests/` +- Run `./clickhouse-test` + +## Example usage + +Run `./clickhouse-test --help` to see available options. + +To run tests without having to create a symlink or mess with `PATH`: + +```bash +./clickhouse-test -c "../../build/dbms/src/Server/clickhouse --client" +``` + +To run a single test, i.e. `00395_nullable`: + +```bash +./clickhouse-test 00395 +``` + diff --git a/docs/en/development/tests.rst b/docs/en/development/tests.rst deleted file mode 100644 index 6638b1358f7..00000000000 --- a/docs/en/development/tests.rst +++ /dev/null @@ -1,38 +0,0 @@ -How to run ClickHouse tests -=========================== - -The ``clickhouse-test`` utility that is used for functional testing is written using Python 2.x. -It also requires you to have some third-party packages: - -.. code-block:: bash - - $ pip install lxml termcolor - - -In a nutshell: - -- Put the ``clickhouse`` program to ``/usr/bin`` (or ``PATH``) -- Create a ``clickhouse-client`` symlink in ``/usr/bin`` pointing to ``clickhouse`` -- Start the ``clickhouse`` server -- ``cd dbms/tests/`` -- Run ``./clickhouse-test`` - - -Example usage -------------- - -Run ``./clickhouse-test --help`` to see available options. - -To run tests without having to create a symlink or mess with ``PATH``: - -.. code-block:: bash - - ./clickhouse-test -c "../../build/dbms/src/Server/clickhouse --client" - - -To run a single test, i.e. ``00395_nullable``: - -.. code-block:: bash - - ./clickhouse-test 00395 - diff --git a/docs/en/dicts/external_dicts.md b/docs/en/dicts/external_dicts.md new file mode 100644 index 00000000000..b99b02bbf57 --- /dev/null +++ b/docs/en/dicts/external_dicts.md @@ -0,0 +1,54 @@ + + +# External dictionaries + +You can add your own dictionaries from various data sources. The data source for a dictionary can be a local text or executable file, an HTTP(s) resource, or another DBMS. For more information, see "[Sources for external dictionaries](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources)". + +ClickHouse: + +> - Fully or partially stores dictionaries in RAM. +- Periodically updates dictionaries and dynamically loads missing values. In other words, dictionaries can be loaded dynamically. + +The configuration of external dictionaries is located in one or more files. The path to the configuration is specified in the [dictionaries_config](../operations/server_settings/settings.md#server_settings-dictionaries_config) parameter. + +Dictionaries can be loaded at server startup or at first use, depending on the [dictionaries_lazy_load](../operations/server_settings/settings.md#server_settings-dictionaries_lazy_load) setting. + +The dictionary config file has the following format: + +```xml + + An optional element with any content. Ignored by the ClickHouse server. + + + /etc/metrika.xml + + + + + + + ... + + + + + +``` + +You can [configure](external_dicts_dict.md#dicts-external_dicts_dict) any number of dictionaries in the same file. The file format is preserved even if there is only one dictionary (i.e. ` - /etc/metrika.xml - - - - - os - - - - - - - /opt/dictionaries/os.tsv - - TabSeparated - - - - - - - - - - cat /opt/dictionaries/os.tsv - - TabSeparated - - - - - - http://[::1]/os.tsv - - TabSeparated - - - - - - - 300 - 360 - - - - - - - - - - - - - - - - Id - - - - - Name - - String - - - - - - ParentID - UInt64 - 0 - - true - - true - - - - - -The dictionary identifier (key attribute) should be a number that fits into UInt64. Also, you can use arbitrary tuples as keys (see section "Dictionaries with complex keys"). Note: you can use complex keys consisting of just one element. This allows using e.g. Strings as dictionary keys. - -There are six ways to store dictionaries in memory. - -flat ----- -This is the most effective method. It works if all keys are smaller than ``500,000``. If a larger key is discovered when creating the dictionary, an exception is thrown and the dictionary is not created. The dictionary is loaded to RAM in its entirety. The dictionary uses the amount of memory proportional to maximum key value. With the limit of 500,000, memory consumption is not likely to be high. All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety. - -hashed ------- -This method is slightly less effective than the first one. The dictionary is also loaded to RAM in its entirety, and can contain any number of items with any identifiers. In practice, it makes sense to use up to tens of millions of items, while there is enough RAM. -All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety. - -cache ------ -This is the least effective method. It is appropriate if the dictionary doesn't fit in RAM. It is a cache of a fixed number of cells, where frequently-used data can be located. MySQL, ClickHouse, executable, http sources are supported, but file sources are not supported. -When searching a dictionary, the cache is searched first. For each data block, all keys not found in the cache (or expired keys) are collected in a package, which is sent to the source with the query ``SELECT attrs... FROM db.table WHERE id IN (k1, k2, ...)``. The received data is then written to the cache. - -range_hashed ------------- -The table lists some data for date ranges, for each key. To give the possibility to extract this data for a given key, for a given date. - -Example: in the table there are discounts for each advertiser in the form: - -.. code-block:: text - - advertiser id discount start date end date value - 123 2015-01-01 2015-01-15 0.15 - 123 2015-01-16 2015-01-31 0.25 - 456 2015-01-01 2015-01-15 0.05 - -Adding layout = range_hashed. -When using such a layout, the structure should have the elements range_min, range_max. - -Example: - -.. code-block:: xml - - - - Id - - - first - - - last - - ... - -These columns must be of type Date. Other types are not yet supported. -The columns indicate a closed date range. - -To work with such dictionaries, dictGetT functions must take one more argument - the date: - -``dictGetT('dict_name', 'attr_name', id, date)`` - -The function takes out the value for this id and for the date range, which includes the transmitted date. If no id is found or the range found is not found for the found id, the default value for the dictionary is returned. - -If there are overlapping ranges, then any suitable one can be used. - -If the range boundary is NULL or is an incorrect date (1900-01-01, 2039-01-01), then the range should be considered open. The range can be open on both sides. - -In the RAM, the data is presented as a hash table with a value in the form of an ordered array of ranges and their corresponding values. - -Example of a dictionary by ranges: - -.. code-block:: xml - - - - xxx - - - xxx - 3306 - xxx - - xxx - 1 - - dicts - xxx
- - - - 300 - 360 - - - - - - - Abcdef - - - StartDate - - - EndDate - - - XXXType - String - - - - - - -ip_trie -------- -The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score. - -Example: in the table there are prefixes matches to AS number and country: - -.. code-block:: text - - prefix asn cca2 - 202.79.32.0/20 17501 NP - 2620:0:870::/48 3856 US - 2a02:6b8:1::/48 13238 RU - 2001:db8::/32 65536 ZZ - - -When using such a layout, the structure should have the "key" element. - -Example: - -.. code-block:: xml - - - - - prefix - String - - - - asn - UInt32 - - - - cca2 - String - ?? - - ... - -These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported. - -For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used: - -``dictGetT('dict_name', 'attr_name', tuple(ip))`` - -The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format: - -``dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))`` - -No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned. - -The data is stored currently in a bitwise trie, it has to fit in memory. - -complex_key_hashed ------------------- - -The same as ``hashed``, but for complex keys. - -complex_key_cache ------------------ - -The same as ``cache``, but for complex keys. - -Notes ------ - -We recommend using the ``flat`` method when possible, or ``hashed``. The speed of the dictionaries is impeccable with this type of memory storage. - -Use the cache method only in cases when it is unavoidable. The speed of the cache depends strongly on correct settings and the usage scenario. A cache type dictionary only works normally for high enough hit rates (recommended 99% and higher). You can view the average hit rate in the system.dictionaries table. Set a large enough cache size. You will need to experiment to find the right number of cells - select a value, use a query to get the cache completely full, look at the memory consumption (this information is in the system.dictionaries table), then proportionally increase the number of cells so that a reasonable amount of memory is consumed. We recommend MySQL as the source for the cache, because ClickHouse doesn't handle requests with random reads very well. - -In all cases, performance is better if you call the function for working with a dictionary after ``GROUP BY``, and if the attribute being fetched is marked as injective. For a dictionary cache, performance improves if you call the function after LIMIT. To do this, you can use a subquery with LIMIT, and call the function with the dictionary from the outside. - -An attribute is called injective if different attribute values correspond to different keys. So when ``GROUP BY`` uses a function that fetches an attribute value by the key, this function is automatically taken out of ``GROUP BY``. - -When updating dictionaries from a file, first the file modification time is checked, and it is loaded only if the file has changed. -When updating from MySQL, for flat and hashed dictionaries, first a ``SHOW TABLE STATUS`` query is made, and the table update time is checked. If it is not NULL, it is compared to the stored time. This works for MyISAM tables, but for InnoDB tables the update time is unknown, so loading from InnoDB is performed on each update. - -For cache dictionaries, the expiration (lifetime) of data in the cache can be set. If more time than 'lifetime' has passed since loading the data in a cell, the cell's value is not used, and it is re-requested the next time it needs to be used. - -If a dictionary couldn't be loaded even once, an attempt to use it throws an exception. -If an error occurred during a request to a cached source, an exception is thrown. -Dictionary updates (other than loading for first use) do not block queries. During updates, the old version of a dictionary is used. If an error occurs during an update, the error is written to the server log, and queries continue using the old version of dictionaries. - -You can view the list of external dictionaries and their status in the system.dictionaries table. - -To use external dictionaries, see the section "Functions for working with external dictionaries". - -Note that you can convert values for a small dictionary by specifying all the contents of the dictionary directly in a ``SELECT`` query (see the section "transform function"). This functionality is not related to external dictionaries. - -Dictionaries with complex keys ------------------------------- - -You can use tuples consisting of fields of arbitrary types as keys. Configure your dictionary with ``complex_key_hashed`` or ``complex_key_cache`` layout in this case. - -Key structure is configured not in the ```` element but in the ```` element. Fields of the key tuple are configured analogously to dictionary attributes. Example: - -.. code-block:: xml - - - - - field1 - String - - - field2 - UInt32 - - ... - - ... - - -When using such dictionary, use a Tuple of field values as a key in dictGet* functions. Example: ``dictGetString('dict_name', 'attr_name', tuple('field1_value', 123))``. diff --git a/docs/en/dicts/external_dicts_dict.md b/docs/en/dicts/external_dicts_dict.md new file mode 100644 index 00000000000..0b9bb4d6299 --- /dev/null +++ b/docs/en/dicts/external_dicts_dict.md @@ -0,0 +1,34 @@ + + +# Configuring an external dictionary + +The dictionary configuration has the following structure: + +```xml + + dict_name + + + + + + + + + + + + + + + + + +``` + +- name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. +- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) – Source of the dictionary. +- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) – Location of the dictionary in memory. +- [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) – Structure of the dictionary. A key and attributes that can be retrieved by this key. +- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) – How frequently to update dictionaries. + diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md new file mode 100644 index 00000000000..e8d4da503e6 --- /dev/null +++ b/docs/en/dicts/external_dicts_dict_layout.md @@ -0,0 +1,222 @@ + + +# Storing dictionaries in memory + +There are [many different ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory. + +We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed. + +Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)" section. + +There are several ways to improve dictionary performance: + +- Call the function for working with the dictionary after `GROUP BY`. +- Mark attributes to extract as injective. An attribute is called injective if different attribute values correspond to different keys. So when `GROUP BY` uses a function that fetches an attribute value by the key, this function is automatically taken out of `GROUP BY`. + +ClickHouse generates an exception for errors with dictionaries. Examples of errors: + +- The dictionary being accessed could not be loaded. +- Error querying a `cached` dictionary. + +You can view the list of external dictionaries and their statuses in the `system.dictionaries` table. + +The configuration looks like this: + +```xml + + + ... + + + + + + ... + + +``` + + + +## Ways to store dictionaries in memory + +- [flat](#dicts-external_dicts_dict_layout-flat) +- [hashed](#dicts-external_dicts_dict_layout-hashed) +- [cache](#dicts-external_dicts_dict_layout-cache) +- [range_hashed](#dicts-external_dicts_dict_layout-range_hashed) +- [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed) +- [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache) + + + +### flat + +The dictionary is completely stored in memory in the form of flat arrays. How much memory does the dictionary use? The amount is proportional to the size of the largest key (in space used). + +The dictionary key has the ` UInt64` type and the value is limited to 500,000. If a larger key is discovered when creating the dictionary, ClickHouse throws an exception and does not create the dictionary. + +All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety. + +This method provides the best performance among all available methods of storing the dictionary. + +Configuration example: + +```xml + + + +``` + + + +### hashed + +The dictionary is completely stored in memory in the form of a hash table. The dictionary can contain any number of elements with any identifiers In practice, the number of keys can reach tens of millions of items. + +All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety. + +Configuration example: + +```xml + + + +``` + + + +### complex_key_hashed + +This type of storage is designed for use with compound [keys](..external_dicts_dict_structure.html/#dicts-external_dicts_dict_structure). It is similar to hashed. + +Configuration example: + +```xml + + + +``` + + + +### range_hashed + +The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values. + +This storage method works the same way as hashed and allows using date/time ranges in addition to the key, if they appear in the dictionary. + +Example: The table contains discounts for each advertiser in the format: + +``` ++------------------+-----------------------------+------------+----------+ | advertiser id | discount start date | discount end date | amount | +==================+=============================+============+==========+ | 123 | 2015-01-01 | 2015-01-15 | 0.15 | +------------------+-----------------------------+------------+----------+ | 123 | 2015-01-16 | 2015-01-31 | 0.25 | +------------------+-----------------------------+------------+----------+ | 456 | 2015-01-01 | 2015-01-15 | 0.05 | +------------------+-----------------------------+------------+----------+ +``` + +To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). + +Example: + +```xml + + + Id + + + first + + + last + + ... +``` + +To work with these dictionaries, you need to pass an additional date argument to the `dictGetT` function: + + dictGetT('dict_name', 'attr_name', id, date) + +This function returns the value for the specified `id`s and the date range that includes the passed date. + +Details of the algorithm: + +- If the ` id` is not found or a range is not found for the ` id`, it returns the default value for the dictionary. +- If there are overlapping ranges, you can use any. +- If the range delimiter is `NULL` or an invalid date (such as 1900-01-01 or 2039-01-01), the range is left open. The range can be open on both sides. + +Configuration example: + +```xml + + + + ... + + + + + + + + Abcdef + + + StartDate + + + EndDate + + + XXXType + String + + + + + + +``` + + + +### cache + +The dictionary is stored in a cache that has a fixed number of cells. These cells contain frequently used elements. + +When searching for a dictionary, the cache is searched first. For each block of data, all keys that are not found in the cache or are outdated are requested from the source using ` SELECT attrs... FROM db.table WHERE id IN (k1, k2, ...)`. The received data is then written to the cache. + +For cache dictionaries, the expiration (lifetime <dicts-external_dicts_dict_lifetime>) of data in the cache can be set. If more time than `lifetime` has passed since loading the data in a cell, the cell's value is not used, and it is re-requested the next time it needs to be used. + +This is the least effective of all the ways to store dictionaries. The speed of the cache depends strongly on correct settings and the usage scenario. A cache type dictionary performs well only when the hit rates are high enough (recommended 99% and higher). You can view the average hit rate in the `system.dictionaries` table. + +To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally. + +Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP. + +Example of settings: + +```xml + + + + 1000000000 + + +``` + +Set a large enough cache size. You need to experiment to select the number of cells: + +1. Set some value. +2. Run queries until the cache is completely full. +3. Assess memory consumption using the `system.dictionaries` table. +4. Increase or decrease the number of cells until the required memory consumption is reached. + +
+ +Do not use ClickHouse as a source, because it is slow to process queries with random reads. + +
+ + + +### complex_key_cache + +This type of storage is designed for use with compound [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`. + diff --git a/docs/en/dicts/external_dicts_dict_lifetime.md b/docs/en/dicts/external_dicts_dict_lifetime.md new file mode 100644 index 00000000000..6431fb3de48 --- /dev/null +++ b/docs/en/dicts/external_dicts_dict_lifetime.md @@ -0,0 +1,59 @@ + + +# Dictionary updates + +ClickHouse periodically updates the dictionaries. The update interval for fully downloaded dictionaries and the invalidation interval for cached dictionaries are defined in the `` tag in seconds. + +Dictionary updates (other than loading for first use) do not block queries. During updates, the old version of a dictionary is used. If an error occurs during an update, the error is written to the server log, and queries continue using the old version of dictionaries. + +Example of settings: + +```xml + + ... + 300 + ... + +``` + +Setting ` 0 ` prevents updating dictionaries. + +You can set a time interval for upgrades, and ClickHouse will choose a uniformly random time within this range. This is necessary in order to distribute the load on the dictionary source when upgrading on a large number of servers. + +Example of settings: + +```xml + + ... + + 300 + 360 + + ... + +``` + +When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): + +> - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. +> - For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. +> - Dictionaries from other sources are updated every time by default. + +For MySQL (InnoDB) and ODBC sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: + +> - The dictionary table must have a field that always changes when the source data is updated. +> - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. The query must be specified in the `` field in the [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) settings. + +Example of settings: + +```xml + + ... + + ... + SELECT update_time FROM dictionary_source where id = 1 + + ... + +``` + diff --git a/docs/en/dicts/external_dicts_dict_sources.md b/docs/en/dicts/external_dicts_dict_sources.md new file mode 100644 index 00000000000..24992f7e7a1 --- /dev/null +++ b/docs/en/dicts/external_dicts_dict_sources.md @@ -0,0 +1,397 @@ + + +# Sources of external dictionaries + +An external dictionary can be connected from many different sources. + +The configuration looks like this: + +```xml + + + ... + + + + + + ... + + ... + +``` + +The source is configured in the `source` section. + +Types of sources (`source_type`): + +- [Local file](#dicts-external_dicts_dict_sources-local_file) +- [Executable file](#dicts-external_dicts_dict_sources-executable) +- [HTTP(s)](#dicts-external_dicts_dict_sources-http) +- [ODBC](#dicts-external_dicts_dict_sources-odbc) +- DBMS + - [MySQL](#dicts-external_dicts_dict_sources-mysql) + - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) + - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) + + + +## Local file + +Example of settings: + +```xml + + + /opt/dictionaries/os.tsv + TabSeparated + + +``` + +Setting fields: + +- `path` – The absolute path to the file. +- `format` – The file format. All the formats described in "[Formats](../formats/index.md#formats)" are supported. + + + +## Executable file + +Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request to the executable file's `STDIN`. + +Example of settings: + +```xml + + + cat /opt/dictionaries/os.tsv + TabSeparated + + +``` + +Setting fields: + +- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). +- `format` – The file format. All the formats described in "[Formats](../formats/index.md#formats)" are supported. + + + +## HTTP(s) + +Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method. + +Example of settings: + +```xml + + + http://[::1]/os.tsv + TabSeparated + + +``` + +In order for ClickHouse to access an HTTPS resource, you must [configure openSSL](../operations/server_settings/settings.md#server_settings-openSSL) in the server configuration. + +Setting fields: + +- `url` – The source URL. +- `format` – The file format. All the formats described in "[Formats](../formats/index.md#formats)" are supported. + + + +## ODBC + +You can use this method to connect any database that has an ODBC driver. + +Example of settings: + +```xml + + DatabaseName + TableName
+ DSN=some_parameters + SQL_QUERY + +``` + +Setting fields: + +- `db` – Name of the database. Omit it if the database name is set in the `` parameters. +- `table` – Name of the table. +- `connection_string` – Connection string. +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime). + +## Example of connecting PostgreSQL + +Ubuntu OS. + +Installing unixODBC and the ODBC driver for PostgreSQL: + + sudo apt-get install -y unixodbc odbcinst odbc-postgresql + +Configuring `/etc/odbc.ini` (или `~/.odbc.ini`): : + + [DEFAULT] + Driver = myconnection + + [myconnection] + Description = PostgreSQL connection to my_db + Driver = PostgreSQL Unicode + Database = my_db + Servername = 127.0.0.1 + UserName = username + Password = password + Port = 5432 + Protocol = 9.3 + ReadOnly = No + RowVersioning = No + ShowSystemTables = No + ConnSettings = + +The dictionary configuration in ClickHouse: + +```xml + + table_name + + + + + DSN=myconnection + postgresql_table
+ + + + 300 + 360 + + + + + + + id + + + some_column + UInt64 + 0 + + + +``` + +You may need to edit `odbc.ini` to specify the full path to the library with the driver `DRIVER=/usr/local/lib/psqlodbcw.so`. + +### Example of connecting MS SQL Server + +Ubuntu OS. + +Installing the driver: : + +``` +sudo apt-get install tdsodbc freetds-bin sqsh +``` + +Configuring the driver: : + +``` + $ cat /etc/freetds/freetds.conf + ... + + [MSSQL] + host = 192.168.56.101 + port = 1433 + tds version = 7.0 + client charset = UTF-8 + + $ cat /etc/odbcinst.ini + ... + + [FreeTDS] + Description = FreeTDS + Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so + Setup = /usr/lib/x86_64-linux-gnu/odbc/libtdsS.so + FileUsage = 1 + UsageCount = 5 + + $ cat ~/.odbc.ini + ... + + [MSSQL] + Description = FreeTDS + Driver = FreeTDS + Servername = MSSQL + Database = test + UID = test + PWD = test + Port = 1433 +``` + +Configuring the dictionary in ClickHouse: + +```xml + + + test + + + dict
+ DSN=MSSQL;UID=test;PWD=test + + + + + 300 + 360 + + + + + + + + + k + + + s + String + + + + + +``` + +## DBMS + + + +### MySQL + +Example of settings: + +```xml + + + 3306 + clickhouse + qwerty + + example01-1 + 1 + + + example01-2 + 1 + + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +Setting fields: + +- `port` – The port on the MySQL server. You can specify it for all replicas, or for each one individually (inside ``). + +- `user` – Name of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). + +- `password` – Password of the MySQL user. You can specify it for all replicas, or for each one individually (inside ``). + +- `replica` – Section of replica configurations. There can be multiple sections. + - `replica/host` – The MySQL host. + + \* `replica/priority` – The replica priority. When attempting to connect, ClickHouse traverses the replicas in order of priority. The lower the number, the higher the priority. + +- `db` – Name of the database. + +- `table` – Name of the table. + +- `where ` – The selection criteria. Optional parameter. + +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime). + +MySQL can be connected on a local host via sockets. To do this, set `host` and `socket`. + +Example of settings: + +```xml + + + localhost + /path/to/socket/file.sock + clickhouse + qwerty + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + + + +### ClickHouse + +Example of settings: + +```xml + + + example01-01-1 + 9000 + default + + default + ids
+ id=10 + + +``` + +Setting fields: + +- `host` – The ClickHouse host. If it is a local host, the query is processed without any network activity. To improve fault tolerance, you can create a [Distributed](../table_engines/distributed.md#table_engines-distributed) table and enter it in subsequent configurations. +- `port` – The port on the ClickHouse server. +- `user` – Name of the ClickHouse user. +- `password` – Password of the ClickHouse user. +- `db` – Name of the database. +- `table` – Name of the table. +- `where ` – The selection criteria. May be omitted. + + + +### MongoDB + +Example of settings: + +```xml + + + localhost + 27017 + + + test + dictionary_source + + +``` + +Setting fields: + +- `host` – The MongoDB host. +- `port` – The port on the MongoDB server. +- `user` – Name of the MongoDB user. +- `password` – Password of the MongoDB user. +- `db` – Name of the database. +- `collection` – Name of the collection. + diff --git a/docs/en/dicts/external_dicts_dict_structure.md b/docs/en/dicts/external_dicts_dict_structure.md new file mode 100644 index 00000000000..ca7c89a912d --- /dev/null +++ b/docs/en/dicts/external_dicts_dict_structure.md @@ -0,0 +1,122 @@ + + +# Dictionary key and fields + +The `` clause describes the dictionary key and fields available for queries. + +Overall structure: + +```xml + + + + Id + + + + + + + ... + + + +``` + +Columns are described in the structure: + +- `` – [Key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key). +- `` – [Data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns. + + + +## Key + +ClickHouse supports the following types of keys: + +- Numeric key. UInt64. Defined in the tag `` . +- Composite key. Set of values of different types. Defined in the tag `` . + +A structure can contain either `` or `` . + +
+ +The key doesn't need to be defined separately in attributes. + +
+ +### Numeric key + +Format: `UInt64`. + +Configuration example: + +```xml + + Id + +``` + +Configuration fields: + +- name – The name of the column with keys. + +### Composite key + +The key can be a `tuple` from any types of fields. The [ layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`. + +
+ +A composite key can also consist of a single element, which makes it possible to use a string as the key, for instance. + +
+ +The key structure is set in the element ``. Key fields are specified in the same format as the dictionary [attributes](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). Example: + +```xml + + + + field1 + String + + + field2 + UInt32 + + ... + +... +``` + +For a query to the `dictGet*` function, a tuple is passed as the key. Example: `dictGetString('dict_name', 'attr_name', tuple('string for field1', num_for_field2))`. + + + +## Attributes + +Configuration example: + +```xml + + ... + + Name + Type + + rand64() + true + true + + +``` + +Configuration fields: + +- `name` – The column name. +- `type` – The column type. Sets the method for interpreting data in the source. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the source table, but it can be uploaded as `String`. +- `null_value` – The default value for a non-existing element. In the example, it is an empty string. +- `expression` – The attribute can be an expression. The tag is not required. +- `hierarchical` – Hierarchical support. Mirrored to the parent identifier. By default, ` false`. +- `injective` Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`. + diff --git a/docs/en/dicts/index.md b/docs/en/dicts/index.md new file mode 100644 index 00000000000..fd055d85703 --- /dev/null +++ b/docs/en/dicts/index.md @@ -0,0 +1,15 @@ +# Dictionaries + +`A dictionary` is a mapping (key `->` attributes) that can be used in a query as functions. +You can think of this as a more convenient and efficient type of JOIN with dimension tables. + +There are built-in (internal) and add-on (external) dictionaries. + +```eval_rst +.. toctree:: + + external_dicts + internal_dicts + +``` + diff --git a/docs/en/dicts/index.rst b/docs/en/dicts/index.rst deleted file mode 100644 index 4ace173a8f7..00000000000 --- a/docs/en/dicts/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -Dictionaries -============ - -A dictionary is a mapping (key -> attributes) that can be used in a query as functions. You can think of this as a more convenient and efficient type of JOIN with dimension tables. - -There are built-in (internal) and add-on (external) dictionaries. - -.. toctree:: - :glob: - - * diff --git a/docs/en/dicts/internal_dicts.md b/docs/en/dicts/internal_dicts.md new file mode 100644 index 00000000000..7ab94d64535 --- /dev/null +++ b/docs/en/dicts/internal_dicts.md @@ -0,0 +1,49 @@ +# Internal dictionaries + +ClickHouse contains a built-in feature for working with a geobase. + +This allows you to: + +- Use a region's ID to get its name in the desired language. +- Use a region's ID to get the ID of a city, area, federal district, country, or continent. +- Check whether a region is part of another region. +- Get a chain of parent regions. + +All the functions support "translocality," the ability to simultaneously use different perspectives on region ownership. For more information, see the section "Functions for working with Yandex.Metrica dictionaries". + +The internal dictionaries are disabled in the default package. +To enable them, uncomment the parameters `path_to_regions_hierarchy_file` and `path_to_regions_names_files` in the server configuration file. + +The geobase is loaded from text files. +If you work at Yandex, you can follow these instructions to create them: + + +Put the regions_hierarchy\*.txt files in the path_to_regions_hierarchy_file directory. This configuration parameter must contain the path to the regions_hierarchy.txt file (the default regional hierarchy), and the other files (regions_hierarchy_ua.txt) must be located in the same directory. + +Put the `regions_names_*.txt` files in the path_to_regions_names_files directory. + +You can also create these files yourself. The file format is as follows: + +`regions_hierarchy*.txt`: TabSeparated (no header), columns: + +- Region ID (UInt32) +- Parent region ID (UInt32) +- Region type (UInt8): 1 - continent, 3 - country, 4 - federal district, 5 - region, 6 - city; other types don't have values. +- Population (UInt32) - Optional column. + +`regions_names_*.txt`: TabSeparated (no header), columns: + +- Region ID (UInt32) +- Region name (String) - Can't contain tabs or line feeds, even escaped ones. + +A flat array is used for storing in RAM. For this reason, IDs shouldn't be more than a million. + +Dictionaries can be updated without restarting the server. However, the set of available dictionaries is not updated. +For updates, the file modification times are checked. If a file has changed, the dictionary is updated. +The interval to check for changes is configured in the 'builtin_dictionaries_reload_interval' parameter. +Dictionary updates (other than loading at first use) do not block queries. During updates, queries use the old versions of dictionaries. If an error occurs during an update, the error is written to the server log, and queries continue using the old version of dictionaries. + +We recommend periodically updating the dictionaries with the geobase. During an update, generate new files and write them to a separate location. When everything is ready, rename them to the files used by the server. + +There are also functions for working with OS identifiers and Yandex.Metrica search engines, but they shouldn't be used. + diff --git a/docs/en/dicts/internal_dicts.rst b/docs/en/dicts/internal_dicts.rst deleted file mode 100644 index 17f706048ce..00000000000 --- a/docs/en/dicts/internal_dicts.rst +++ /dev/null @@ -1,45 +0,0 @@ -Internal dictionaries ---------------------- - -ClickHouse contains a built-in feature for working with a geobase. - -This allows you to: - * Use a region's ID to get its name in the desired language. - * Use a region's ID to get the ID of a city, area, federal district, country, or continent. - * Check whether a region is part of another region. - * Get a chain of parent regions. - -All the functions support "translocality", the ability to simultaneously use different perspectives on region ownership. For more information, see the section "Functions for working with Yandex.Metrica dictionaries". - -The internal dictionaries are disabled in the default package. -To enable them, uncomment the parameters ``path_to_regions_hierarchy_file`` and ``path_to_regions_names_files`` in the server config file. - -The geobase is loaded from text files. -If you are Yandex employee, to create them, use the following instructions: -https://github.yandex-team.ru/raw/Metrika/ClickHouse_private/master/doc/create_embedded_geobase_dictionaries.txt - -Put the regions_hierarchy*.txt files in the path_to_regions_hierarchy_file directory. This configuration parameter must contain the path to the regions_hierarchy.txt file (the default regional hierarchy), and the other files (regions_hierarchy_ua.txt) must be located in the same directory. - -Put the regions_names_*.txt files in the path_to_regions_names_files directory. - -You can also create these files yourself. The file format is as follows: - -``regions_hierarchy*.txt``: TabSeparated (no header), columns: - * Region ID (UInt32) - * Parent region ID (UInt32) - * Region type (UInt8): 1 - continent, 3 - country, 4 - federal district, 5 - region, 6 - city; other types don't have values. - * Population (UInt32) - Optional column. - -``regions_names_*.txt``: TabSeparated (no header), columns: - * Region ID (UInt32) - * Region name (String) - Can't contain tabs or line breaks, even escaped ones. - -A flat array is used for storing in RAM. For this reason, IDs shouldn't be more than a million. - -Dictionaries can be updated without the server restart. However, the set of available dictionaries is not updated. For updates, the file modification times are checked. If a file has changed, the dictionary is updated. -The interval to check for changes is configured in the 'builtin_dictionaries_reload_interval' parameter. -Dictionary updates (other than loading at first use) do not block queries. During updates, queries use the old versions of dictionaries. If an error occurs during an update, the error is written to the server log, while queries continue using the old version of dictionaries. - -We recommend periodically updating the dictionaries with the geobase. During an update, generate new files and write them to a separate location. When everything is ready, rename them to the files used by the server. - -There are also functions for working with OS identifiers and Yandex.Metrica search engines, but they shouldn't be used. diff --git a/docs/en/formats/capnproto.md b/docs/en/formats/capnproto.md new file mode 100644 index 00000000000..0d482e20887 --- /dev/null +++ b/docs/en/formats/capnproto.md @@ -0,0 +1,26 @@ + + +# CapnProto + +Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack. + +Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query. + +```sql +SELECT SearchPhrase, count() AS c FROM test.hits + GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' +``` + +Where `schema.capnp` looks like this: + +``` +struct Message { + SearchPhrase @0 :Text; + c @1 :Uint64; +} +``` + +Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration. + +Deserialization is effective and usually doesn't increase the system load. + diff --git a/docs/en/formats/capnproto.rst b/docs/en/formats/capnproto.rst deleted file mode 100644 index 5b6c4b79796..00000000000 --- a/docs/en/formats/capnproto.rst +++ /dev/null @@ -1,29 +0,0 @@ -CapnProto ---------- - -Cap'n Proto is a binary message format. Like Protocol Buffers and Thrift (but unlike JSON or MessagePack), Cap'n Proto messages are strongly-typed and not self-describing. Due to this, it requires a ``schema`` setting to specify schema file and the root object. The schema is parsed on runtime and cached for each SQL statement. - -.. code-block:: sql - - SELECT SearchPhrase, count() AS c FROM test.hits - GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' - -When the `schema.capnp` schema file looks like: - -.. code-block:: text - - struct Message { - SearchPhrase @0 :Text; - c @1 :Uint64; - } - -The schema files are located in the path specified in the configuration file: - -.. code-block:: xml - - - format_schemas/ - -Deserialization is almost as efficient as the binary rows format, with typically zero allocation overhead per message. - -You can use this format as an efficient exchange message format in your data processing pipeline. diff --git a/docs/en/formats/csv.md b/docs/en/formats/csv.md new file mode 100644 index 00000000000..7aacf9c1485 --- /dev/null +++ b/docs/en/formats/csv.md @@ -0,0 +1,10 @@ +# CSV -15 + +Comma Separated Values format ([RFC](https://tools.ietf.org/html/rfc4180)). + +When formatting, rows are enclosed in double quotes. A double quote inside a string is output as two double quotes in a row. There are no other rules for escaping characters. Date and date-time are enclosed in double quotes. Numbers are output without quotes. Values ​​are separated by commas. Rows are separated using the Unix line feed (LF). Arrays are serialized in CSV as follows: first the array is serialized to a string as in TabSeparated format, and then the resulting string is output to CSV in double quotes. Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost). + +When parsing, all values can be parsed either with or without quotes. Both double and single quotes are supported. Rows can also be arranged without quotes. In this case, they are parsed up to a comma or line feed (CR or LF). In violation of the RFC, when parsing rows without quotes, the leading and trailing spaces and tabs are ignored. For the line feed, Unix (LF), Windows (CR LF) and Mac OS Classic (CR LF) are all supported. + +The CSV format supports the output of totals and extremes the same way as `TabSeparated`. + diff --git a/docs/en/formats/csv.rst b/docs/en/formats/csv.rst deleted file mode 100644 index 87753d37579..00000000000 --- a/docs/en/formats/csv.rst +++ /dev/null @@ -1,10 +0,0 @@ -CSV ---- - -Comma separated values (`RFC `_). - -String values are output in double quotes. Double quote inside a string is output as two consecutive double quotes. That's all escaping rules. Date and DateTime values are output in double quotes. Numbers are output without quotes. Fields are delimited by commas. Rows are delimited by unix newlines (LF). Arrays are output in following way: first, array are serialized to String (as in TabSeparated or Values formats), and then the String value are output in double quotes. Tuples are narrowed and serialized as separate columns. - -During parsing, values could be enclosed or not enclosed in quotes. Supported both single and double quotes. In particular, Strings could be represented without quotes - in that case, they are parsed up to comma or newline (CR or LF). Contrary to RFC, in case of parsing strings without quotes, leading and trailing spaces and tabs are ignored. As line delimiter, both Unix (LF), Windows (CR LF) or Mac OS Classic (LF CR) variants are supported. - -CSV format supports output of totals and extremes similar to TabSeparated format. diff --git a/docs/en/formats/csvwithnames.md b/docs/en/formats/csvwithnames.md new file mode 100644 index 00000000000..8933bcb33a0 --- /dev/null +++ b/docs/en/formats/csvwithnames.md @@ -0,0 +1,4 @@ +# CSVWithNames + +Also prints the header row, similar to `TabSeparatedWithNames`. + diff --git a/docs/en/formats/csvwithnames.rst b/docs/en/formats/csvwithnames.rst deleted file mode 100644 index 524adbccf37..00000000000 --- a/docs/en/formats/csvwithnames.rst +++ /dev/null @@ -1,4 +0,0 @@ -CSVWithNames ------------- - -Also contains header, similar to ``TabSeparatedWithNames``. diff --git a/docs/en/formats/index.md b/docs/en/formats/index.md new file mode 100644 index 00000000000..ad2b192d99f --- /dev/null +++ b/docs/en/formats/index.md @@ -0,0 +1,13 @@ + + +# Formats + +The format determines how data is returned to you after SELECTs (how it is written and formatted by the server), and how it is accepted for INSERTs (how it is read and parsed by the server). + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/formats/index.rst b/docs/en/formats/index.rst deleted file mode 100644 index 2947bd2d93e..00000000000 --- a/docs/en/formats/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Formats -======= - -The format determines how data is given (written by server as output) to you after SELECTs, and how it is accepted (read by server as input) for INSERTs. - -.. toctree:: - :glob: - - * diff --git a/docs/en/formats/json.md b/docs/en/formats/json.md new file mode 100644 index 00000000000..3b8354f0b88 --- /dev/null +++ b/docs/en/formats/json.md @@ -0,0 +1,86 @@ +# JSON + +Outputs data in JSON format. Besides data tables, it also outputs column names and types, along with some additional information: the total number of output rows, and the number of rows that could have been output if there weren't a LIMIT. Example: + +```sql +SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON +``` + +```json +{ + "meta": + [ + { + "name": "SearchPhrase", + "type": "String" + }, + { + "name": "c", + "type": "UInt64" + } + ], + + "data": + [ + { + "SearchPhrase": "", + "c": "8267016" + }, + { + "SearchPhrase": "bathroom interior design", + "c": "2166" + }, + { + "SearchPhrase": "yandex", + "c": "1655" + }, + { + "SearchPhrase": "spring 2014 fashion", + "c": "1549" + }, + { + "SearchPhrase": "freeform photo", + "c": "1480" + } + ], + + "totals": + { + "SearchPhrase": "", + "c": "8873898" + }, + + "extremes": + { + "min": + { + "SearchPhrase": "", + "c": "1480" + }, + "max": + { + "SearchPhrase": "", + "c": "8267016" + } + }, + + "rows": 5, + + "rows_before_limit_at_least": 141137 +} +``` + +The JSON is compatible with JavaScript. To ensure this, some characters are additionally escaped: the slash ` /` is escaped as ` \/`; alternative line breaks ` U+2028` and ` U+2029`, which break some browsers, are escaped as ` \uXXXX`. ASCII control characters are escaped: backspace, form feed, line feed, carriage return, and horizontal tab are replaced with `\b`, `\f`, `\n`, `\r`, `\t` , as well as the remaining bytes in the 00-1F range using `\uXXXX` sequences. Invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. For compatibility with JavaScript, Int64 and UInt64 integers are enclosed in double quotes by default. To remove the quotes, you can set the configuration parameter output_format_json_quote_64bit_integers to 0. + +`rows` – The total number of output rows. + +`rows_before_limit_at_least` The minimal number of rows there would have been without LIMIT. Output only if the query contains LIMIT. +If the query contains GROUP BY, rows_before_limit_at_least is the exact number of rows there would have been without a LIMIT. + +`totals` – Total values (when using WITH TOTALS). + +`extremes` – Extreme values (when extremes is set to 1). + +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). +See also the JSONEachRow format. + diff --git a/docs/en/formats/json.rst b/docs/en/formats/json.rst deleted file mode 100644 index 4cf03028ba8..00000000000 --- a/docs/en/formats/json.rst +++ /dev/null @@ -1,87 +0,0 @@ -JSON ----- - -Outputs data in JSON format. Besides data tables, it also outputs column names and types, along with some additional information - the total number of output rows, and the number of rows that could have been output if there weren't a LIMIT. Example: - -.. code-block:: sql - - SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTALS ORDER BY c DESC LIMIT 5 FORMAT JSON - -.. code-block:: json - - { - "meta": - [ - { - "name": "SearchPhrase", - "type": "String" - }, - { - "name": "c", - "type": "UInt64" - } - ], - - "data": - [ - { - "SearchPhrase": "", - "c": "8267016" - }, - { - "SearchPhrase": "интерьер ванной комнаты", - "c": "2166" - }, - { - "SearchPhrase": "яндекс", - "c": "1655" - }, - { - "SearchPhrase": "весна 2014 мода", - "c": "1549" - }, - { - "SearchPhrase": "фриформ фото", - "c": "1480" - } - ], - - "totals": - { - "SearchPhrase": "", - "c": "8873898" - }, - - "extremes": - { - "min": - { - "SearchPhrase": "", - "c": "1480" - }, - "max": - { - "SearchPhrase": "", - "c": "8267016" - } - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 - } - -JSON is compatible with JavaScript. For this purpose, certain symbols are additionally escaped: the forward slash ``/`` is escaped as ``\/``; alternative line breaks ``U+2028`` and ``U+2029``, which don't work in some browsers, are escaped as \uXXXX-sequences. ASCII control characters are escaped: backspace, form feed, line feed, carriage return, and horizontal tab as ``\b``, ``\f``, ``\n``, ``\r``, and ``\t`` respectively, along with the rest of the bytes from the range 00-1F using \uXXXX-sequences. Invalid UTF-8 sequences are changed to the replacement character ``�`` and, thus, the output text will consist of valid UTF-8 sequences. UInt64 and Int64 numbers are output in double quotes for compatibility with JavaScript. - -``rows`` - The total number of output rows. - -``rows_before_limit_at_least`` - The minimal number of rows there would have been without a LIMIT. Output only if the query contains LIMIT. - -If the query contains GROUP BY, ``rows_before_limit_at_least`` is the exact number of rows there would have been without a LIMIT. - -``totals`` - Total values (when using WITH TOTALS). - -``extremes`` - Extreme values (when extremes is set to 1). - -This format is only appropriate for outputting a query result, not for parsing. -See JSONEachRow format for INSERT queries. diff --git a/docs/en/formats/jsoncompact.md b/docs/en/formats/jsoncompact.md new file mode 100644 index 00000000000..e4ce0867bc2 --- /dev/null +++ b/docs/en/formats/jsoncompact.md @@ -0,0 +1,46 @@ +# JSONCompact + +Differs from JSON only in that data rows are output in arrays, not in objects. + +Example: + +```json +{ + "meta": + [ + { + "name": "SearchPhrase", + "type": "String" + }, + { + "name": "c", + "type": "UInt64" + } + ], + + "data": + [ + ["", "8267016"], + ["bathroom interior design", "2166"], + ["yandex", "1655"], + ["spring 2014 fashion", "1549"], + ["freeform photos", "1480"] + ], + + "totals": ["","8873898"], + + "extremes": + { + "min": ["","1480"], + "max": ["","8267016"] + }, + + "rows": 5, + + "rows_before_limit_at_least": 141137 +} +``` + +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). +See also the `JSONEachRow` format. + diff --git a/docs/en/formats/jsoncompact.rst b/docs/en/formats/jsoncompact.rst deleted file mode 100644 index 33afb76a173..00000000000 --- a/docs/en/formats/jsoncompact.rst +++ /dev/null @@ -1,46 +0,0 @@ -JSONCompact ------------ - -Differs from ``JSON`` only in that data rows are output in arrays, not in objects. - -Example: - -.. code-block:: text - - { - "meta": - [ - { - "name": "SearchPhrase", - "type": "String" - }, - { - "name": "c", - "type": "UInt64" - } - ], - - "data": - [ - ["", "8267016"], - ["bath interiors", "2166"], - ["yandex", "1655"], - ["spring 2014 fashion", "1549"], - ["freeform photo", "1480"] - ], - - "totals": ["","8873898"], - - "extremes": - { - "min": ["","1480"], - "max": ["","8267016"] - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 - } - -This format is only appropriate for outputting a query result, not for parsing. -See ``JSONEachRow`` format for INSERT queries. diff --git a/docs/en/formats/jsoneachrow.md b/docs/en/formats/jsoneachrow.md new file mode 100644 index 00000000000..510da367fe5 --- /dev/null +++ b/docs/en/formats/jsoneachrow.md @@ -0,0 +1,21 @@ +# JSONEachRow + +Outputs data as separate JSON objects for each row (newline delimited JSON). + +```json +{"SearchPhrase":"","count()":"8267016"} +{"SearchPhrase":"bathroom interior design","count()":"2166"} +{"SearchPhrase":"yandex","count()":"1655"} +{"SearchPhrase":"spring 2014 fashion","count()":"1549"} +{"SearchPhrase":"freeform photo","count()":"1480"} +{"SearchPhrase":"angelina jolie","count()":"1245"} +{"SearchPhrase":"omsk","count()":"1112"} +{"SearchPhrase":"photos of dog breeds","count()":"1091"} +{"SearchPhrase":"curtain design","count()":"1064"} +{"SearchPhrase":"baku","count()":"1000"} +``` + +Unlike the JSON format, there is no substitution of invalid UTF-8 sequences. Any set of bytes can be output in the rows. This is necessary so that data can be formatted without losing any information. Values are escaped in the same way as for JSON. + +For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. Whitespace between elements is ignored. If a comma is placed after the objects, it is ignored. Objects don't necessarily have to be separated by new lines. + diff --git a/docs/en/formats/jsoneachrow.rst b/docs/en/formats/jsoneachrow.rst deleted file mode 100644 index 9be09e4f224..00000000000 --- a/docs/en/formats/jsoneachrow.rst +++ /dev/null @@ -1,25 +0,0 @@ -JSONEachRow ------------ - -If put in SELECT query, displays data in newline delimited JSON (JSON objects separated by \\n character) format. -If put in INSERT query, expects this kind of data as input. - -.. code-block:: text - - {"SearchPhrase":"","count()":"8267016"} - {"SearchPhrase":"bathroom interior","count()":"2166"} - {"SearchPhrase":"yandex","count()":"1655"} - {"SearchPhrase":"spring 2014 fashion","count()":"1549"} - {"SearchPhrase":"free-form photo","count()":"1480"} - {"SearchPhrase":"Angelina Jolie","count()":"1245"} - {"SearchPhrase":"omsk","count()":"1112"} - {"SearchPhrase":"photos of dog breeds","count()":"1091"} - {"SearchPhrase":"curtain design","count()":"1064"} - {"SearchPhrase":"baku","count()":"1000"} - -Unlike JSON format, there are no replacements of invalid UTF-8 sequences. There can be arbitrary amount of bytes in a line. -This is done in order to avoid data loss during formatting. Values are displayed analogous to JSON format. - -In INSERT queries JSON data can be supplied with arbitrary order of columns (JSON key-value pairs). It is also possible to omit values in which case the default value of the column is inserted. N.B. when using JSONEachRow format, complex default values are not supported, so when omitting a column its value will be zeros or empty string depending on its type. - -Space characters between JSON objects are skipped. Between objects there can be a comma which is ignored. Newline character is not a mandatory separator for objects. diff --git a/docs/en/formats/native.rst b/docs/en/formats/native.md similarity index 67% rename from docs/en/formats/native.rst rename to docs/en/formats/native.md index bea3eae72d8..c3cd06c7bb0 100644 --- a/docs/en/formats/native.rst +++ b/docs/en/formats/native.md @@ -1,6 +1,6 @@ -Native ------- +# Native -The most efficient format. Data is written and read by blocks in binary format. For each block, the number of rows, number of columns, column names and types, and parts of columns in this block are recorded one after another. In other words, this format is "columnar" - it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +The most efficient format. Data is written and read by blocks in binary format. For each block, the number of rows, number of columns, column names and types, and parts of columns in this block are recorded one after another. In other words, this format is "columnar" – it doesn't convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. You can use this format to quickly generate dumps that can only be read by the ClickHouse DBMS. It doesn't make sense to work with this format yourself. + diff --git a/docs/en/formats/null.md b/docs/en/formats/null.md new file mode 100644 index 00000000000..119f9d445d4 --- /dev/null +++ b/docs/en/formats/null.md @@ -0,0 +1,5 @@ +# Null + +Nothing is output. However, the query is processed, and when using the command-line client, data is transmitted to the client. This is used for tests, including productivity testing. +Obviously, this format is only appropriate for output, not for parsing. + diff --git a/docs/en/formats/null.rst b/docs/en/formats/null.rst deleted file mode 100644 index a3ea6d40e51..00000000000 --- a/docs/en/formats/null.rst +++ /dev/null @@ -1,4 +0,0 @@ -Null ----- - -Nothing is output. However, the query is processed, and when using the command-line client, data is transmitted to the client. This is used for tests, including productivity testing. Obviously, this format is only appropriate for outputting a query result, not for parsing. diff --git a/docs/en/formats/pretty.md b/docs/en/formats/pretty.md new file mode 100644 index 00000000000..df31e29381d --- /dev/null +++ b/docs/en/formats/pretty.md @@ -0,0 +1,37 @@ +# Pretty + +Outputs data as Unicode-art tables, also using ANSI-escape sequences for setting colors in the terminal. +A full grid of the table is drawn, and each row occupies two lines in the terminal. +Each result block is output as a separate table. This is necessary so that blocks can be output without buffering results (buffering would be necessary in order to pre-calculate the visible width of all the values). +To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). + +The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the PrettyCompact format): + +```sql +SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact +``` + +```text +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1406958 │ +│ 2014-03-18 │ 1383658 │ +│ 2014-03-19 │ 1405797 │ +│ 2014-03-20 │ 1353623 │ +│ 2014-03-21 │ 1245779 │ +│ 2014-03-22 │ 1031592 │ +│ 2014-03-23 │ 1046491 │ +└────────────┴─────────┘ + +Totals: +┌──EventDate─┬───────c─┐ +│ 0000-00-00 │ 8873898 │ +└────────────┴─────────┘ + +Extremes: +┌──EventDate─┬───────c─┐ +│ 2014-03-17 │ 1031592 │ +│ 2014-03-23 │ 1406958 │ +└────────────┴─────────┘ +``` + diff --git a/docs/en/formats/pretty.rst b/docs/en/formats/pretty.rst deleted file mode 100644 index 696c759a4ae..00000000000 --- a/docs/en/formats/pretty.rst +++ /dev/null @@ -1,36 +0,0 @@ -Pretty ------- - -Writes data as Unicode-art tables, also using ANSI-escape sequences for setting colors in the terminal. -A full grid of the table is drawn, and each row occupies two lines in the terminal. Each result block is output as a separate table. This is necessary so that blocks can be output without buffering results (buffering would be necessary in order to pre-calculate the visible width of all the values). -To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10,000" is printed. -This format is only appropriate for outputting a query result, not for parsing. - -The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the PrettyCompact format): - -.. code-block:: sql - - SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact - -.. code-block:: text - - ┌──EventDate─┬───────c─┐ - │ 2014-03-17 │ 1406958 │ - │ 2014-03-18 │ 1383658 │ - │ 2014-03-19 │ 1405797 │ - │ 2014-03-20 │ 1353623 │ - │ 2014-03-21 │ 1245779 │ - │ 2014-03-22 │ 1031592 │ - │ 2014-03-23 │ 1046491 │ - └────────────┴─────────┘ - - Totals: - ┌──EventDate─┬───────c─┐ - │ 0000-00-00 │ 8873898 │ - └────────────┴─────────┘ - - Extremes: - ┌──EventDate─┬───────c─┐ - │ 2014-03-17 │ 1031592 │ - │ 2014-03-23 │ 1406958 │ - └────────────┴─────────┘ diff --git a/docs/en/formats/prettycompact.md b/docs/en/formats/prettycompact.md new file mode 100644 index 00000000000..52b0dd87b2b --- /dev/null +++ b/docs/en/formats/prettycompact.md @@ -0,0 +1,5 @@ +# PrettyCompact + +Differs from `Pretty` in that the grid is drawn between rows and the result is more compact. +This format is used by default in the command-line client in interactive mode. + diff --git a/docs/en/formats/prettycompact.rst b/docs/en/formats/prettycompact.rst deleted file mode 100644 index bae790f1aaf..00000000000 --- a/docs/en/formats/prettycompact.rst +++ /dev/null @@ -1,4 +0,0 @@ -PrettyCompact -------------- - -Differs from ``Pretty`` in that the grid is drawn between rows and the result is more compact. This format is used by default in the command-line client in interactive mode. diff --git a/docs/en/formats/prettycompactmonoblock.md b/docs/en/formats/prettycompactmonoblock.md new file mode 100644 index 00000000000..6c1fd9a0df8 --- /dev/null +++ b/docs/en/formats/prettycompactmonoblock.md @@ -0,0 +1,4 @@ +# PrettyCompactMonoBlock + +Differs from `PrettyCompact` in that up to 10,000 rows are buffered, then output as a single table, not by blocks. + diff --git a/docs/en/formats/prettycompactmonoblock.rst b/docs/en/formats/prettycompactmonoblock.rst deleted file mode 100644 index 35e18b045b1..00000000000 --- a/docs/en/formats/prettycompactmonoblock.rst +++ /dev/null @@ -1,4 +0,0 @@ -PrettyCompactMonoBlock ----------------------- - -Differs from ``PrettyCompact`` in that up to 10,000 rows are buffered, then output as a single table, not by blocks. diff --git a/docs/en/formats/prettynoescapes.md b/docs/en/formats/prettynoescapes.md new file mode 100644 index 00000000000..51c6ec2e5bf --- /dev/null +++ b/docs/en/formats/prettynoescapes.md @@ -0,0 +1,20 @@ +# PrettyNoEscapes + +Differs from Pretty in that ANSI-escape sequences aren't used. This is necessary for displaying this format in a browser, as well as for using the 'watch' command-line utility. + +Example: + +```bash +watch -n1 "clickhouse-client --query='SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'" +``` + +You can use the HTTP interface for displaying in the browser. + +## PrettyCompactNoEscapes + +The same as the previous setting. + +## PrettySpaceNoEscapes + +The same as the previous setting. + diff --git a/docs/en/formats/prettynoescapes.rst b/docs/en/formats/prettynoescapes.rst deleted file mode 100644 index 49997422ce8..00000000000 --- a/docs/en/formats/prettynoescapes.rst +++ /dev/null @@ -1,20 +0,0 @@ -PrettyNoEscapes ---------------- - -Differs from Pretty in that ANSI-escape sequences aren't used. This is necessary for displaying this format in a browser, as well as for using the 'watch' command-line utility. - -Example: - -.. code-block:: text - - watch -n1 "clickhouse-client --query='SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'" - -You can use the HTTP interface for displaying in the browser. - -PrettyCompactNoEscapes ----------------------- -The same. - -PrettySpaceNoEscapes --------------------- -The same. diff --git a/docs/en/formats/prettyspace.md b/docs/en/formats/prettyspace.md new file mode 100644 index 00000000000..5c352a6ee54 --- /dev/null +++ b/docs/en/formats/prettyspace.md @@ -0,0 +1,4 @@ +# PrettySpace + +Differs from `PrettyCompact` in that whitespace (space characters) is used instead of the grid. + diff --git a/docs/en/formats/prettyspace.rst b/docs/en/formats/prettyspace.rst deleted file mode 100644 index 3d8aec8a934..00000000000 --- a/docs/en/formats/prettyspace.rst +++ /dev/null @@ -1,4 +0,0 @@ -PrettySpace ------------ - -Differs from ``PrettyCompact`` in that whitespace (space characters) is used instead of the grid. diff --git a/docs/en/formats/rowbinary.md b/docs/en/formats/rowbinary.md new file mode 100644 index 00000000000..bc8479332ba --- /dev/null +++ b/docs/en/formats/rowbinary.md @@ -0,0 +1,13 @@ +# RowBinary + +Formats and parses data by row in binary format. Rows and values are listed consecutively, without separators. +This format is less efficient than the Native format, since it is row-based. + +Integers use fixed-length little endian representation. For example, UInt64 uses 8 bytes. +DateTime is represented as UInt32 containing the Unix timestamp as the value. +Date is represented as a UInt16 object that contains the number of days since 1970-01-01 as the value. +String is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string. +FixedString is represented simply as a sequence of bytes. + +Arrays are represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the array elements in order. + diff --git a/docs/en/formats/rowbinary.rst b/docs/en/formats/rowbinary.rst deleted file mode 100644 index b87a27706a8..00000000000 --- a/docs/en/formats/rowbinary.rst +++ /dev/null @@ -1,13 +0,0 @@ -RowBinary ---------- - -Writes data by row in binary format. Rows and values are listed consecutively, without separators. -This format is less efficient than the Native format, since it is row-based. - -Numbers is written in little endian, fixed width. For example, UInt64 takes 8 bytes. -DateTime is written as UInt32 with unix timestamp value. -Date is written as UInt16 with number of days since 1970-01-01 in value. -String is written as length in varint (unsigned `LEB128 `_) format and then bytes of string. -FixedString is written as just its bytes. -Array is written as length in varint (unsigned `LEB128 `_) format and then all elements, contiguously - diff --git a/docs/en/formats/tabseparated.md b/docs/en/formats/tabseparated.md new file mode 100644 index 00000000000..6f4cba79e2f --- /dev/null +++ b/docs/en/formats/tabseparated.md @@ -0,0 +1,59 @@ +# TabSeparated + +In TabSeparated format, data is written by row. Each row contains values separated by tabs. Each value is follow by a tab, except the last value in the row, which is followed by a line feed. Strictly Unix line feeds are assumed everywhere. The last row also must contain a line feed at the end. Values are written in text format, without enclosing quotation marks, and with special characters escaped. + +Integer numbers are written in decimal form. Numbers can contain an extra "+" character at the beginning (ignored when parsing, and not recorded when formatting). Non-negative numbers can't contain the negative sign. When reading, it is allowed to parse an empty string as a zero, or (for signed types) a string consisting of just a minus sign as a zero. Numbers that do not fit into the corresponding data type may be parsed as a different number, without an error message. + +Floating-point numbers are written in decimal form. The dot is used as the decimal separator. Exponential entries are supported, as are 'inf', '+inf', '-inf', and 'nan'. An entry of floating-point numbers may begin or end with a decimal point. +During formatting, accuracy may be lost on floating-point numbers. +During parsing, it is not strictly required to read the nearest machine-representable number. + +Dates are written in YYYY-MM-DD format and parsed in the same format, but with any characters as separators. +Dates with times are written in the format YYYY-MM-DD hh:mm:ss and parsed in the same format, but with any characters as separators. +This all occurs in the system time zone at the time the client or server starts (depending on which one formats data). For dates with times, daylight saving time is not specified. So if a dump has times during daylight saving time, the dump does not unequivocally match the data, and parsing will select one of the two times. +During a read operation, incorrect dates and dates with times can be parsed with natural overflow or as null dates and times, without an error message. + +As an exception, parsing dates with times is also supported in Unix timestamp format, if it consists of exactly 10 decimal digits. The result is not time zone-dependent. The formats YYYY-MM-DD hh:mm:ss and NNNNNNNNNN are differentiated automatically. + +Strings are output with backslash-escaped special characters. The following escape sequences are used for output: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`. Parsing also supports the sequences `\a`, `\v`, and `\xHH` (hex escape sequences) and any `\c` sequences, where `c` is any character (these sequences are converted to `c`). Thus, reading data supports formats where a line feed can be written as `\n` or `\`, or as a line feed. For example, the string `Hello world` with a line feed between the words instead of a space can be parsed in any of the following variations: + +```text +Hello\nworld + +Hello\ +world +``` + +The second variant is supported because MySQL uses it when writing tab-separated dumps. + +The minimum set of characters that you need to escape when passing data in TabSeparated format: tab, line feed (LF) and backslash. + +Only a small set of symbols are escaped. You can easily stumble onto a string value that your terminal will ruin in output. + +Arrays are written as a list of comma-separated values in square brackets. Number items in the array are fomratted as normally, but dates, dates with times, and strings are written in single quotes with the same escaping rules as above. + +The TabSeparated format is convenient for processing data using custom programs and scripts. It is used by default in the HTTP interface, and in the command-line client's batch mode. This format also allows transferring data between different DBMSs. For example, you can get a dump from MySQL and upload it to ClickHouse, or vice versa. + +The TabSeparated format supports outputting total values (when using WITH TOTALS) and extreme values (when 'extremes' is set to 1). In these cases, the total values and extremes are output after the main data. The main result, total values, and extremes are separated from each other by an empty line. Example: + +```sql +SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` +``` + +```text +2014-03-17 1406958 +2014-03-18 1383658 +2014-03-19 1405797 +2014-03-20 1353623 +2014-03-21 1245779 +2014-03-22 1031592 +2014-03-23 1046491 + +0000-00-00 8873898 + +2014-03-17 1031592 +2014-03-23 1406958 +``` + +This format is also available under the name `TSV`. + diff --git a/docs/en/formats/tabseparated.rst b/docs/en/formats/tabseparated.rst deleted file mode 100644 index f1f6da4b747..00000000000 --- a/docs/en/formats/tabseparated.rst +++ /dev/null @@ -1,59 +0,0 @@ -TabSeparated ------------- - -In TabSeparated format, data is written by row. Each row contains values separated by tabs. Each value is follow by a tab, except the last value in the row, which is followed by a line break. Strictly Unix line breaks are assumed everywhere. The last row also must contain a line break at the end. Values are written in text format, without enclosing quotation marks, and with special characters escaped. - -Numbers are written in decimal form. Numbers may contain an extra "+" symbol at the beginning (but it is not recorded during an output). Non-negative numbers can't contain the negative sign. When parsing, it is allowed to parse an empty string as a zero, or (for signed types) a string consisting of just a minus sign as a zero. Numbers that do not fit into the corresponding data type may be parsed as a different number, without an error message. - -Floating-point numbers are formatted in decimal form. The dot is used as the decimal separator. Exponential entries are supported, as are 'inf', '+inf', '-inf', and 'nan'. An entry of floating-point numbers may begin or end with a decimal point. -During formatting, accuracy may be lost on floating-point numbers. -During parsing, a result is not necessarily the nearest machine-representable number. - -Dates are formatted in YYYY-MM-DD format and parsed in the same format, but with any characters as separators. -DateTimes are formatted in the format YYYY-MM-DD hh:mm:ss and parsed in the same format, but with any characters as separators. -This all occurs in the system time zone at the time the client or server starts (depending on which one formats data). For DateTimes, daylight saving time is not specified. So if a dump has times during daylight saving time, the dump does not unequivocally match the data, and parsing will select one of the two times. -During a parsing operation, incorrect dates and dates with times can be parsed with natural overflow or as null dates and times, without an error message. - -As an exception, parsing DateTime is also supported in Unix timestamp format, if it consists of exactly 10 decimal digits. The result is not time zone-dependent. The formats ``YYYY-MM-DD hh:mm:ss`` and ``NNNNNNNNNN`` are differentiated automatically. - -Strings are parsed and formatted with backslash-escaped special characters. The following escape sequences are used while formatting: ``\b``, ``\f``, ``\r``, ``\n``, ``\t``, ``\0``, ``\'``, and ``\\``. For parsing, also supported \a, \v and \xHH (hex escape sequence) and any sequences of the type \c where c is any character (these sequences are converted to c). This means that parsing supports formats where a line break can be written as \n or as \ and a line break. For example, the string 'Hello world' with a line break between the words instead of a space can be retrieved in any of the following variations: - -.. code-block:: text - - Hello\nworld - - Hello\ - world - -The second variant is supported because MySQL uses it when writing tab-separated dumps. - -Only a small set of symbols are escaped. You can easily stumble onto a string value that your terminal will ruin in output. - -Minimum set of symbols that you must escape in TabSeparated format is tab, newline (LF) and backslash. - -Arrays are formatted as a list of comma-separated values in square brackets. Number items in the array are formatted as normally, but dates, dates with times, and strings are formatted in single quotes with the same escaping rules as above. - -The TabSeparated format is convenient for processing data using custom programs and scripts. It is used by default in the HTTP interface, and in the command-line client's batch mode. This format also allows transferring data between different DBMSs. For example, you can get a dump from MySQL and upload it to ClickHouse, or vice versa. - -The TabSeparated format supports outputting total values (when using WITH TOTALS) and extreme values (when 'extremes' is set to 1). In these cases, the total values and extremes are output after the main data. The main result, total values, and extremes are separated from each other by an empty line. Example: - -.. code-block:: sql - - SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated - -.. code-block:: text - - 2014-03-17 1406958 - 2014-03-18 1383658 - 2014-03-19 1405797 - 2014-03-20 1353623 - 2014-03-21 1245779 - 2014-03-22 1031592 - 2014-03-23 1046491 - - 0000-00-00 8873898 - - 2014-03-17 1031592 - 2014-03-23 1406958 - -It's also available as ``TSV``. diff --git a/docs/en/formats/tabseparatedraw.md b/docs/en/formats/tabseparatedraw.md new file mode 100644 index 00000000000..526742f838d --- /dev/null +++ b/docs/en/formats/tabseparatedraw.md @@ -0,0 +1,7 @@ +# TabSeparatedRaw + +Differs from `TabSeparated` format in that the rows are written without escaping. +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). + +This format is also available under the name ` TSVRaw`. + diff --git a/docs/en/formats/tabseparatedraw.rst b/docs/en/formats/tabseparatedraw.rst deleted file mode 100644 index 9b8db1346e1..00000000000 --- a/docs/en/formats/tabseparatedraw.rst +++ /dev/null @@ -1,7 +0,0 @@ -TabSeparatedRaw ---------------- - -Differs from the ``TabSeparated`` format in that the rows are formatted without escaping. -This format is only appropriate for outputting a query result, but not for parsing data to insert into a table. - -It's also available as ``TSVRaw``. diff --git a/docs/en/formats/tabseparatedwithnames.md b/docs/en/formats/tabseparatedwithnames.md new file mode 100644 index 00000000000..e8d42690bf4 --- /dev/null +++ b/docs/en/formats/tabseparatedwithnames.md @@ -0,0 +1,8 @@ +# TabSeparatedWithNames + +Differs from the `TabSeparated` format in that the column names are written in the first row. +During parsing, the first row is completely ignored. You can't use column names to determine their position or to check their correctness. +(Support for parsing the header row may be added in the future.) + +This format is also available under the name ` TSVWithNames`. + diff --git a/docs/en/formats/tabseparatedwithnames.rst b/docs/en/formats/tabseparatedwithnames.rst deleted file mode 100644 index 17c75719bef..00000000000 --- a/docs/en/formats/tabseparatedwithnames.rst +++ /dev/null @@ -1,8 +0,0 @@ -TabSeparatedWithNames ---------------------- - -Differs from the TabSeparated format in that the column names are output in the first row. -For parsing, the first row is completely ignored. You can't use column names to determine their position or to check their correctness. -(Support for using header while parsing could be added in future.) - -It's also available as ``TSVWithNames``. diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.md b/docs/en/formats/tabseparatedwithnamesandtypes.md new file mode 100644 index 00000000000..461481fb586 --- /dev/null +++ b/docs/en/formats/tabseparatedwithnamesandtypes.md @@ -0,0 +1,7 @@ +# TabSeparatedWithNamesAndTypes + +Differs from the `TabSeparated` format in that the column names are written to the first row, while the column types are in the second row. +During parsing, the first and second rows are completely ignored. + +This format is also available under the name ` TSVWithNamesAndTypes`. + diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.rst b/docs/en/formats/tabseparatedwithnamesandtypes.rst deleted file mode 100644 index b903dcb339a..00000000000 --- a/docs/en/formats/tabseparatedwithnamesandtypes.rst +++ /dev/null @@ -1,7 +0,0 @@ -TabSeparatedWithNamesAndTypes ------------------------------ - -Differs from the ``TabSeparated`` format in that the column names are output to the first row, while the column types are in the second row. -For parsing, the first and second rows are completely ignored. - -It's also available as ``TSVWithNamesAndTypes``. diff --git a/docs/en/formats/tskv.md b/docs/en/formats/tskv.md new file mode 100644 index 00000000000..12de5ccf6e6 --- /dev/null +++ b/docs/en/formats/tskv.md @@ -0,0 +1,23 @@ +# TSKV + +Similar to TabSeparated, but outputs a value in name=value format. Names are escaped the same way as in TabSeparated format, and the = symbol is also escaped. + +```text +SearchPhrase= count()=8267016 +SearchPhrase=bathroom interior design count()=2166 +SearchPhrase=yandex count()=1655 +SearchPhrase=spring 2014 fashion count()=1549 +SearchPhrase=freeform photos count()=1480 +SearchPhrase=angelina jolia count()=1245 +SearchPhrase=omsk count()=1112 +SearchPhrase=photos of dog breeds count()=1091 +SearchPhrase=curtain design count()=1064 +SearchPhrase=baku count()=1000 +``` + +When there is a large number of small columns, this format is ineffective, and there is generally no reason to use it. It is used in some departments of Yandex. + +Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. + +Parsing allows the presence of the additional field `tskv` without the equal sign or a value. This field is ignored. + diff --git a/docs/en/formats/tskv.rst b/docs/en/formats/tskv.rst deleted file mode 100644 index 2db287d6274..00000000000 --- a/docs/en/formats/tskv.rst +++ /dev/null @@ -1,21 +0,0 @@ -TSKV ----- - -Similar to TabSeparated, but displays data in name=value format. Names are displayed just as in TabSeparated. Additionally, a ``=`` symbol is displayed. - -.. code-block:: text - - SearchPhrase= count()=8267016 - SearchPhrase=bathroom interior count()=2166 - SearchPhrase=yandex count()=1655 - SearchPhrase=spring 2014 fashion count()=1549 - SearchPhrase=free-form photo count()=1480 - SearchPhrase=Angelina Jolie count()=1245 - SearchPhrase=omsk count()=1112 - SearchPhrase=photos of dog breeds count()=1091 - SearchPhrase=curtain design count()=1064 - SearchPhrase=baku count()=1000 - -In case of many small columns this format is obviously not effective and there usually is no reason to use it. This format is supported because it is used for some cases in Yandex. - -Format is supported both for input and output. In INSERT queries data can be supplied with arbitrary order of columns. It is also possible to omit values in which case the default value of the column is inserted. N.B. when using TSKV format, complex default values are not supported, so when omitting a column its value will be zeros or empty string depending on its type. diff --git a/docs/en/formats/values.md b/docs/en/formats/values.md new file mode 100644 index 00000000000..9ce9bcf6e04 --- /dev/null +++ b/docs/en/formats/values.md @@ -0,0 +1,9 @@ +# Values + +Prints every row in brackets. Rows are separated by commas. There is no comma after the last row. The values inside the brackets are also comma-separated. Numbers are output in decimal format without quotes. Arrays are output in square brackets. Strings, dates, and dates with times are output in quotes. Escaping rules and parsing are similar to the TabSeparated format. During formatting, extra spaces aren't inserted, but during parsing, they are allowed and skipped (except for spaces inside array values, which are not allowed). + +The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. + +This is the format that is used in `INSERT INTO t VALUES ...` +Но вы также можете использовать его для форматирования результатов запросов. + diff --git a/docs/en/formats/values.rst b/docs/en/formats/values.rst deleted file mode 100644 index f1062d98a2d..00000000000 --- a/docs/en/formats/values.rst +++ /dev/null @@ -1,9 +0,0 @@ -Values ------- - -Prints every row in parentheses. Rows are separated by commas. There is no comma after the last row. The values inside the parentheses are also comma-separated. Numbers are output in decimal format without quotes. Arrays are output in square brackets. Strings, dates, and dates with times are output in quotes. Escaping rules and parsing are same as in the TabSeparated format. During formatting, extra spaces aren't inserted, but during parsing, they are allowed and skipped (except for spaces inside array values, which are not allowed). - -Minimum set of symbols that you must escape in Values format is single quote and backslash. - -This is the format that is used in ``INSERT INTO t VALUES ...`` -But you can also use it for query result. diff --git a/docs/en/formats/vertical.rst b/docs/en/formats/vertical.md similarity index 57% rename from docs/en/formats/vertical.rst rename to docs/en/formats/vertical.md index fa09851255e..71ec816a3f2 100644 --- a/docs/en/formats/vertical.rst +++ b/docs/en/formats/vertical.md @@ -1,5 +1,5 @@ -Vertical --------- +# Vertical Prints each value on a separate line with the column name specified. This format is convenient for printing just one or a few rows, if each row consists of a large number of columns. -This format is only appropriate for outputting a query result, not for parsing. +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). + diff --git a/docs/en/formats/xml.md b/docs/en/formats/xml.md new file mode 100644 index 00000000000..f91adec9356 --- /dev/null +++ b/docs/en/formats/xml.md @@ -0,0 +1,73 @@ +# XML + +XML format is suitable only for output, not for parsing. Example: + +```xml + + + + + + SearchPhrase + String + + + count() + UInt64 + + + + + + + 8267016 + + + bathroom interior design + 2166 + + + yandex + 1655 + + + spring 2014 fashion + 1549 + + + freeform photos + 1480 + + + angelina jolie + 1245 + + + omsk + 1112 + + + photos of dog breeds + 1091 + + + curtain design + 1064 + + + baku + 1000 + + + 10 + 141137 + +``` + +If the column name does not have an acceptable format, just 'field' is used as the element name. In general, the XML structure follows the JSON structure. +Just as for JSON, invalid UTF-8 sequences are changed to the replacement character � so the output text will consist of valid UTF-8 sequences. + +In string values, the characters `<` and `&` are escaped as `<` and `&`. + +Arrays are output as `HelloWorld...`,and tuples as `HelloWorld...`. + diff --git a/docs/en/formats/xml.rst b/docs/en/formats/xml.rst deleted file mode 100644 index c6d3000698c..00000000000 --- a/docs/en/formats/xml.rst +++ /dev/null @@ -1,74 +0,0 @@ -XML ---- - -XML format is supported only for displaying data, not for INSERTS. Example: - -.. code-block:: xml - - - - - - - SearchPhrase - String - - - count() - UInt64 - - - - - - - 8267016 - - - bathroom interior - 2166 - - - yandex> - 1655 - - - spring 2014 fashion - 1549 - - - free-form photo - 1480 - - - Angelina Jolie - 1245 - - - omsk - 1112 - - - photos of dog breeds - 1091 - - - curtain design - 1064 - - - baku - 1000 - - - 10 - 141137 - - -If name of a column contains some unacceptable character, field is used as a name. In other aspects XML uses JSON structure. -As in case of JSON, invalid UTF-8 sequences are replaced by replacement character � so displayed text will only contain valid UTF-8 sequences. - -In string values ``<`` and ``&`` are displayed as ``<`` and ``&``. - -Arrays are displayed as ``HelloWorld...``, -and tuples as ``HelloWorld...``. diff --git a/docs/en/functions/arithmetic_functions.md b/docs/en/functions/arithmetic_functions.md new file mode 100644 index 00000000000..e457fe6684e --- /dev/null +++ b/docs/en/functions/arithmetic_functions.md @@ -0,0 +1,75 @@ +# Arithmetic functions + +For all arithmetic functions, the result type is calculated as the smallest number type that the result fits in, if there is such a type. The minimum is taken simultaneously based on the number of bits, whether it is signed, and whether it floats. If there are not enough bits, the highest bit type is taken. + +Example: + +```sql +SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 + 0 + 0) +``` + +```text +┌─toTypeName(0)─┬─toTypeName(plus(0, 0))─┬─toTypeName(plus(plus(0, 0), 0))─┬─toTypeName(plus(plus(plus(0, 0), 0), 0))─┐ +│ UInt8 │ UInt16 │ UInt32 │ UInt64 │ +└───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘ +``` + +Arithmetic functions work for any pair of types from UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, or Float64. + +Overflow is produced the same way as in C++. + +## plus(a, b), a + b operator + +Calculates the sum of the numbers. +You can also add integer numbers with a date or date and time. In the case of a date, adding an integer means adding the corresponding number of days. For a date with time, it means adding the corresponding number of seconds. + +## minus(a, b), a - b operator + +Calculates the difference. The result is always signed. + +You can also calculate integer numbers from a date or date with time. The idea is the same – see above for 'plus'. + +## multiply(a, b), a \* b operator + +Calculates the product of the numbers. + +## divide(a, b), a / b operator + +Calculates the quotient of the numbers. The result type is always a floating-point type. +It is not integer division. For integer division, use the 'intDiv' function. +When dividing by zero you get 'inf', '-inf', or 'nan'. + +## intDiv(a, b) + +Calculates the quotient of the numbers. Divides into integers, rounding down (by the absolute value). +An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. + +## intDivOrZero(a, b) + +Differs from 'intDiv' in that it returns zero when dividing by zero or when dividing a minimal negative number by minus one. + +## modulo(a, b), a % b operator + +Calculates the remainder after division. +If arguments are floating-point numbers, they are pre-converted to integers by dropping the decimal portion. +The remainder is taken in the same sense as in C++. Truncated division is used for negative numbers. +An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. + +## negate(a), -a operator + +Calculates a number with the reverse sign. The result is always signed. + +## abs(a) + +Calculates the absolute value of the number (a). That is, if a < 0, it returns -a. For unsigned types it doesn't do anything. For signed integer types, it returns an unsigned number. + +## gcd(a, b) + +Returns the greatest common divisor of the numbers. +An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. + +## lcm(a, b) + +Returns the least common multiple of the numbers. +An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. + diff --git a/docs/en/functions/arithmetic_functions.rst b/docs/en/functions/arithmetic_functions.rst deleted file mode 100644 index 213471f378c..00000000000 --- a/docs/en/functions/arithmetic_functions.rst +++ /dev/null @@ -1,76 +0,0 @@ -Arithmetic functions -==================== - -For all arithmetic functions, the result type is calculated as the smallest number type that the result fits in, if there is such a type. The minimum is taken simultaneously based on the number of bits, whether it is signed, and whether it floats. If there are not enough bits, the highest bit type is taken. - -Example: - -.. code-block:: sql - - SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 + 0 + 0) - -.. code-block:: text - - ┌─toTypeName(0)─┬─toTypeName(plus(0, 0))─┬─toTypeName(plus(plus(0, 0), 0))─┬─toTypeName(plus(plus(plus(0, 0), 0), 0))─┐ - │ UInt8 │ UInt16 │ UInt32 │ UInt64 │ - └───────────────┴────────────────────────┴─────────────────────────────────┴──────────────────────────────────────────┘ - -Arithmetic functions work for any pair of types from UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, or Float64. - -Overflow is produced the same way as in C++. - - -plus(a, b), a + b operator --------------------------- -Calculates the sum of the numbers. -You can also add whole numbers with a date or date and time. In the case of a date, adding a whole number means adding the corresponding number of days. For a date with time, it means adding the corresponding number of seconds. - -minus(a, b), a - b operator ---------------------------- -Calculates the difference. The result is always signed. - -You can also calculate whole numbers from a date or date with time. The idea is the same - see above for 'plus'. - -multiply(a, b), a * b operator ------------------------------- -Calculates the product of the numbers. - -divide(a, b), a / b operator ----------------------------- -Calculates the quotient of the numbers. The result type is always a floating-point type. -It is not integer division. For integer division, use the 'intDiv' function. -When dividing by zero you get 'inf', '-inf', or 'nan'. - -intDiv(a, b) ------------- -Calculates the quotient of the numbers. Divides into integers, rounding down (by the absolute value). -When dividing by zero or when dividing a minimal negative number by minus one, an exception is thrown. - -intDivOrZero(a, b) ------------------- -Differs from 'intDiv' in that it returns zero when dividing by zero or when dividing a minimal negative number by minus one. - -modulo(a, b), a % b operator ----------------------------- -Calculates the remainder after division. -If arguments are floating-point numbers, they are pre-converted to integers by dropping the decimal portion. The remainder is taken in the same sense as in C++. Truncated division is used for negative numbers. -An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. - -negate(a), -a operator ----------------------- -Calculates a number with the reverse sign. The result is always signed. - -abs(a) ------- -Calculates the absolute value of the number 'a'. That is, if a< 0, it returns -a. -For unsigned types, it doesn't do anything. For signed integer types, it returns an unsigned number. - -gcd(a, b) ---------- -Calculates the greatest common divisor of the numbers. -When dividing by zero or when dividing a minimal negative number by minus one, an exception is thrown. - -lcm(a, b) ---------- -Calculates the least common multiple of the numbers. -When dividing by zero or when dividing a minimal negative number by minus one, an exception is thrown. diff --git a/docs/en/functions/array_functions.md b/docs/en/functions/array_functions.md new file mode 100644 index 00000000000..7b0b614c171 --- /dev/null +++ b/docs/en/functions/array_functions.md @@ -0,0 +1,323 @@ +# Functions for working with arrays + +## empty + +Returns 1 for an empty array, or 0 for a non-empty array. +The result type is UInt8. +The function also works for strings. + +## notEmpty + +Returns 0 for an empty array, or 1 for a non-empty array. +The result type is UInt8. +The function also works for strings. + +## length + +Returns the number of items in the array. +The result type is UInt64. +The function also works for strings. + +## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 + +## emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64 + +## emptyArrayFloat32, emptyArrayFloat64 + +## emptyArrayDate, emptyArrayDateTime + +## emptyArrayString + +Accepts zero arguments and returns an empty array of the appropriate type. + +## emptyArrayToSingle + +Accepts an empty array and returns a one-element array that is equal to the default value. + +## range(N) + +Returns an array of numbers from 0 to N-1. +Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block. + +## array(x1, ...), оператор \[x1, ...\] + +Creates an array from the function arguments. +The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray\*' function described above). +Returns an 'Array(T)' type result, where 'T' is the smallest common type out of the passed arguments. + +## arrayConcat + +Combines arrays passed as arguments. + +``` +arrayConcat(arrays) +``` + +**Arguments** + +- `arrays` – Arrays of comma-separated `[values]`. + +**Example** + +```sql +SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res +``` + +``` +┌─res───────────┐ +│ [1,2,3,4,5,6] │ +└───────────────┘ +``` + +## arrayElement(arr, n), operator arr[n] + +Get the element with the index 'n' from the array 'arr'.'n' must be any integer type. +Indexes in an array begin from one. +Negative indexes are supported. In this case, it selects the corresponding element numbered from the end. For example, 'arr\[-1\]' is the last item in the array. + +If the index falls outside of the bounds of an array, it returns some default value (0 for numbers, an empty string for strings, etc.). + +## has(arr, elem) + +Checks whether the 'arr' array has the 'elem' element. +Returns 0 if the the element is not in the array, or 1 if it is. + +## indexOf(arr, x) + +Returns the index of the 'x' element (starting from 1) if it is in the array, or 0 if it is not. + +## countEqual(arr, x) + +Returns the number of elements in the array equal to x. Equivalent to arrayCount (elem-> elem = x, arr). + +## arrayEnumerate(arr) + +Returns the array \[1, 2, 3, ..., length (arr) \] + +This function is normally used with ARRAY JOIN. It allows counting something just once for each array after applying ARRAY JOIN. Example: + +```sql +SELECT + count() AS Reaches, + countIf(num = 1) AS Hits +FROM test.hits +ARRAY JOIN + GoalsReached, + arrayEnumerate(GoalsReached) AS num +WHERE CounterID = 160656 +LIMIT 10 +``` + +```text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` + +In this example, Reaches is the number of conversions (the strings received after applying ARRAY JOIN), and Hits is the number of pageviews (strings before ARRAY JOIN). In this particular case, you can get the same result in an easier way: + +```sql +SELECT + sum(length(GoalsReached)) AS Reaches, + count() AS Hits +FROM test.hits +WHERE (CounterID = 160656) AND notEmpty(GoalsReached) +``` + +```text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` + +This function can also be used in higher-order functions. For example, you can use it to get array indexes for elements that match a condition. + +## arrayEnumerateUniq(arr, ...) + +Returns an array the same size as the source array, indicating for each element what its position is among elements with the same value. +For example: arrayEnumerateUniq(\[10, 20, 10, 30\]) = \[1, 1, 2, 1\]. + +This function is useful when using ARRAY JOIN and aggregation of array elements. +Example: + +```sql +SELECT + Goals.ID AS GoalID, + sum(Sign) AS Reaches, + sumIf(Sign, num = 1) AS Visits +FROM test.visits +ARRAY JOIN + Goals, + arrayEnumerateUniq(Goals.ID) AS num +WHERE CounterID = 160656 +GROUP BY GoalID +ORDER BY Reaches DESC +LIMIT 10 +``` + +```text +┌──GoalID─┬─Reaches─┬─Visits─┐ +│ 53225 │ 3214 │ 1097 │ +│ 2825062 │ 3188 │ 1097 │ +│ 56600 │ 2803 │ 488 │ +│ 1989037 │ 2401 │ 365 │ +│ 2830064 │ 2396 │ 910 │ +│ 1113562 │ 2372 │ 373 │ +│ 3270895 │ 2262 │ 812 │ +│ 1084657 │ 2262 │ 345 │ +│ 56599 │ 2260 │ 799 │ +│ 3271094 │ 2256 │ 812 │ +└─────────┴─────────┴────────┘ +``` + +In this example, each goal ID has a calculation of the number of conversions (each element in the Goals nested data structure is a goal that was reached, which we refer to as a conversion) and the number of sessions. Without ARRAY JOIN, we would have counted the number of sessions as sum(Sign). But in this particular case, the rows were multiplied by the nested Goals structure, so in order to count each session one time after this, we apply a condition to the value of the arrayEnumerateUniq(Goals.ID) function. + +The arrayEnumerateUniq function can take multiple arrays of the same size as arguments. In this case, uniqueness is considered for tuples of elements in the same positions in all the arrays. + +```sql +SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res +``` + +```text +┌─res───────────┐ +│ [1,2,1,1,2,1] │ +└───────────────┘ +``` + +This is necessary when using ARRAY JOIN with a nested data structure and further aggregation across multiple elements in this structure. + +## arrayPopBack + +Removes the last item from the array. + +``` +arrayPopBack(array) +``` + +**Arguments** + +- `array` – Array. + +**Example** + +```sql +SELECT arrayPopBack([1, 2, 3]) AS res +``` + +``` +┌─res───┐ +│ [1,2] │ +└───────┘ +``` + +## arrayPopFront + +Removes the first item from the array. + +``` +arrayPopFront(array) +``` + +**Arguments** + +- `array` – Array. + +**Example** + +```sql +SELECT arrayPopFront([1, 2, 3]) AS res +``` + +``` +┌─res───┐ +│ [2,3] │ +└───────┘ +``` + +## arrayPushBack + +Adds one item to the end of the array. + +``` +arrayPushBack(array, single_value) +``` + +**Arguments** + +- `array` – Array. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". + +**Example** + +```sql +SELECT arrayPushBack(['a'], 'b') AS res +``` + +``` +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` + +## arrayPushFront + +Adds one element to the beginning of the array. + +``` +arrayPushFront(array, single_value) +``` + +**Arguments** + +- `array` – Array. +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". + +**Example** + +```sql +SELECT arrayPushBack(['b'], 'a') AS res +``` + +``` +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` + +## arraySlice + +Returns a slice of the array. + +``` +arraySlice(array, offset[, length]) +``` + +**Arguments** + +- `array` – Array of data. +- `offset` – Offset from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. +- `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`. + +**Example** + +```sql +SELECT arraySlice([1, 2, 3, 4, 5], 2, 3) AS res +``` + +``` +┌─res─────┐ +│ [2,3,4] │ +└─────────┘ +``` + +## arrayUniq(arr, ...) + +If one argument is passed, it counts the number of different elements in the array. +If multiple arguments are passed, it counts the number of different tuples of elements at corresponding positions in multiple arrays. + +If you want to get a list of unique items in an array, you can use arrayReduce('groupUniqArray', arr). + +## arrayJoin(arr) + +A special function. See the section ["ArrayJoin function"](array_join.md#functions_arrayjoin). + diff --git a/docs/en/functions/array_functions.rst b/docs/en/functions/array_functions.rst deleted file mode 100644 index c1e061f51c3..00000000000 --- a/docs/en/functions/array_functions.rst +++ /dev/null @@ -1,181 +0,0 @@ -Functions for working with arrays ---------------------------------- - -empty -~~~~~ -Returns 1 for an empty array, or 0 for a non-empty array. -The result type is UInt8. -The function also works for strings. - -notEmpty -~~~~~~~~ -Returns 0 for an empty array, or 1 for a non-empty array. -The result type is UInt8. -The function also works for strings. - -length -~~~~~~ -Returns the number of items in the array. -The result type is UInt64. -The function also works for strings. - -emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -emptyArrayFloat32, emptyArrayFloat64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -emptyArrayDate, emptyArrayDateTime -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -emptyArrayString -~~~~~~~~~~~~~~~~ -Accepts zero arguments and returns an empty array of the appropriate type. - -emptyArrayToSingle -~~~~~~~~~~~~~~~~~~ -Accepts an empty array as argument and returns an array of one element equal to the default value. - -range(N) -~~~~~~~~ -Returns an array of numbers from 0 to N-1. -Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block. - -array(x1, ...), operator [x1, ...] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates an array from the function arguments. -The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray*' function described above). -Returns an 'Array(T)' type result, where 'T' is the smallest common type out of the passed arguments. - -arrayElement(arr, n), operator arr[n] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the element with the index 'n' from the array 'arr'. -'n' should be any integer type. -Indexes in an array begin from one. -Negative indexes are supported - in this case, it selects the corresponding element numbered from the end. For example, 'arr[-1]' is the last item in the array. - -If the index goes beyond the array bounds, a default value is returned (0 for numbers, an empty string for strings, etc.). - -has(arr, elem) -~~~~~~~~~~~~~~ -Checks whether the 'arr' array has the 'elem' element. -Returns 0 if the the element is not in the array, or 1 if it is. - -indexOf(arr, x) -~~~~~~~~~~~~~~~ -Returns the index of the 'x' element (starting from 1) if it is in the array, or 0 if it is not. - -countEqual(arr, x) -~~~~~~~~~~~~~~~~~~ -Returns the number of elements in the array equal to 'x'. Equivalent to ``arrayCount(elem -> elem = x, arr)``. - -arrayEnumerate(arr) -~~~~~~~~~~~~~~~~~~~ -Returns the array ``[1, 2, 3, ..., length(arr)]`` - -This function is normally used together with ARRAY JOIN. It allows counting something just once for each array after applying ARRAY JOIN. Example: - -.. code-block:: sql - - SELECT - count() AS Reaches, - countIf(num = 1) AS Hits - FROM test.hits - ARRAY JOIN - GoalsReached, - arrayEnumerate(GoalsReached) AS num - WHERE CounterID = 160656 - LIMIT 10 - -.. code-block:: text - - ┌─Reaches─┬──Hits─┐ - │ 95606 │ 31406 │ - └─────────┴───────┘ - -In this example, Reaches is the number of conversions (the strings received after applying ARRAY JOIN), and Hits is the number of pageviews (strings before ARRAY JOIN). In this particular case, you can get the same result in an easier way: - -.. code-block:: sql - - SELECT - sum(length(GoalsReached)) AS Reaches, - count() AS Hits - FROM test.hits - WHERE (CounterID = 160656) AND notEmpty(GoalsReached) - -.. code-block:: text - - ┌─Reaches─┬──Hits─┐ - │ 95606 │ 31406 │ - └─────────┴───────┘ - -This function can also be used in higher-order functions. For example, you can use it to get array indexes for elements that match a condition. - -arrayEnumerateUniq(arr, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns an array the same size as the source array, indicating for each element what its position is among elements with the same value. -For example: ``arrayEnumerateUniq([10, 20, 10, 30]) = [1, 1, 2, 1]``. - -This function is useful when using ARRAY JOIN and aggregation of array elements. Example: - -.. code-block:: sql - - SELECT - Goals.ID AS GoalID, - sum(Sign) AS Reaches, - sumIf(Sign, num = 1) AS Visits - FROM test.visits - ARRAY JOIN - Goals, - arrayEnumerateUniq(Goals.ID) AS num - WHERE CounterID = 160656 - GROUP BY GoalID - ORDER BY Reaches DESC - LIMIT 10 - -.. code-block:: text - - ┌──GoalID─┬─Reaches─┬─Visits─┐ - │ 53225 │ 3214 │ 1097 │ - │ 2825062 │ 3188 │ 1097 │ - │ 56600 │ 2803 │ 488 │ - │ 1989037 │ 2401 │ 365 │ - │ 2830064 │ 2396 │ 910 │ - │ 1113562 │ 2372 │ 373 │ - │ 3270895 │ 2262 │ 812 │ - │ 1084657 │ 2262 │ 345 │ - │ 56599 │ 2260 │ 799 │ - │ 3271094 │ 2256 │ 812 │ - └─────────┴─────────┴────────┘ - -In this example, each goal ID has a calculation of the number of conversions (each element in the Goals nested data structure is a goal that was reached, which we refer to as a conversion) and the number of sessions. -Without ARRAY JOIN, we would have counted the number of sessions as ``sum(Sign)``. But in this particular case, the rows were multiplied by the nested Goals structure, so in order to count each session one time after this, -we apply a condition to the value of the ``arrayEnumerateUniq(Goals.ID)`` function. - -The arrayEnumerateUniq function can take multiple arrays of the same size as arguments. In this case, uniqueness is considered for tuples of elements in the same positions in all the arrays. - -.. code-block:: sql - - SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res - -.. code-block:: text - - ┌─res───────────┐ - │ [1,2,1,1,2,1] │ - └───────────────┘ - -This is necessary when using ARRAY JOIN with a nested data structure and further aggregation across multiple elements in this structure. - -arrayUniq(arr, ...) -~~~~~~~~~~~~~~~~~~~ -If a single array is passed, returns a number of unique elements in that array. -If multiple arrays of the same size are passed as arguments to the function, returns a number of unique tuples of elements in the same positions in all the arrays. - -If you need an array of the unique elements, you can use ``arrayReduce('groupUniqArray', arr)``. - -arrayJoin(arr) -~~~~~~~~~~~~~~ -A special function. See the section "arrayJoin function". diff --git a/docs/en/functions/array_join.md b/docs/en/functions/array_join.md new file mode 100644 index 00000000000..6e18f8203c0 --- /dev/null +++ b/docs/en/functions/array_join.md @@ -0,0 +1,31 @@ + + +# arrayJoin function + +This is a very unusual function. + +Normal functions don't change a set of rows, but just change the values in each row (map). +Aggregate functions compress a set of rows (fold or reduce). +The 'arrayJoin' function takes each row and generates a set of rows (unfold). + +This function takes an array as an argument, and propagates the source row to multiple rows for the number of elements in the array. +All the values in columns are simply copied, except the values in the column where this function is applied; it is replaced with the corresponding array value. + +A query can use multiple `arrayJoin` functions. In this case, the transformation is performed multiple times. + +Note the ARRAY JOIN syntax in the SELECT query, which provides broader possibilities. + +Example: + +```sql +SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src +``` + +```text +┌─dst─┬─\'Hello\'─┬─src─────┐ +│ 1 │ Hello │ [1,2,3] │ +│ 2 │ Hello │ [1,2,3] │ +│ 3 │ Hello │ [1,2,3] │ +└─────┴───────────┴─────────┘ +``` + diff --git a/docs/en/functions/array_join.rst b/docs/en/functions/array_join.rst deleted file mode 100644 index 27847854ecb..00000000000 --- a/docs/en/functions/array_join.rst +++ /dev/null @@ -1,32 +0,0 @@ -arrayJoin function ------------------- -This is a very unusual function. - -Normal functions don't change a set of rows, but just change the values in each row (map). Aggregate functions compress a set of rows (fold or reduce). -The 'arrayJoin' function takes each row and generates a set of rows (unfold). - -This function takes an array as an argument, and propagates the source row to multiple rows for the number of elements in the array. -All the values in columns are simply copied, except the values in the column where this function is applied - it is replaced with the corresponding array value. - -A query can use multiple 'arrayJoin' functions. In this case, the transformation is performed multiple times. - -Note the ARRAY JOIN syntax in the SELECT query, which provides broader possibilities. - -Example: - -.. code-block:: sql - - :) SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src - - SELECT - arrayJoin([1, 2, 3] AS src) AS dst, - 'Hello', - src - -.. code-block:: text - - ┌─dst─┬─\'Hello\'─┬─src─────┐ - │ 1 │ Hello │ [1,2,3] │ - │ 2 │ Hello │ [1,2,3] │ - │ 3 │ Hello │ [1,2,3] │ - └─────┴───────────┴─────────┘ diff --git a/docs/en/functions/bit_functions.rst b/docs/en/functions/bit_functions.md similarity index 63% rename from docs/en/functions/bit_functions.rst rename to docs/en/functions/bit_functions.md index b3223e588aa..523413f200a 100644 --- a/docs/en/functions/bit_functions.rst +++ b/docs/en/functions/bit_functions.md @@ -1,24 +1,18 @@ -Bit functions -------------- +# Bit functions Bit functions work for any pair of types from UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, or Float64. The result type is an integer with bits equal to the maximum bits of its arguments. If at least one of the arguments is signed, the result is a signed number. If an argument is a floating-point number, it is cast to Int64. -bitAnd(a, b) -~~~~~~~~~~~~ +## bitAnd(a, b) -bitOr(a, b) -~~~~~~~~~~~ +## bitOr(a, b) -bitXor(a, b) -~~~~~~~~~~~~ +## bitXor(a, b) -bitNot(a) -~~~~~~~~~ +## bitNot(a) -bitShiftLeft(a, b) -~~~~~~~~~~~~~~~~~~ +## bitShiftLeft(a, b) + +## bitShiftRight(a, b) -bitShiftRight(a, b) -~~~~~~~~~~~~~~~~~~~ diff --git a/docs/en/functions/comparison_functions.md b/docs/en/functions/comparison_functions.md new file mode 100644 index 00000000000..e37642d42ed --- /dev/null +++ b/docs/en/functions/comparison_functions.md @@ -0,0 +1,31 @@ +# Comparison functions + +Comparison functions always return 0 or 1 (Uint8). + +The following types can be compared: + +- numbers +- strings and fixed strings +- dates +- dates with times + +within each group, but not between different groups. + +For example, you can't compare a date with a string. You have to use a function to convert the string to a date, or vice versa. + +Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character. + +Note: Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 > -1. This behavior changed in version 1.1.54134 and is now mathematically correct. + +## equals, a = b and a == b operator + +## notEquals, a ! operator= b and a `<>` b + +## less, `< operator` + +## greater, `> operator` + +## lessOrEquals, `<= operator` + +## greaterOrEquals, `>= operator` + diff --git a/docs/en/functions/comparison_functions.rst b/docs/en/functions/comparison_functions.rst deleted file mode 100644 index 3a3d6c240f1..00000000000 --- a/docs/en/functions/comparison_functions.rst +++ /dev/null @@ -1,36 +0,0 @@ -Comparison functions --------------------- - -Comparison functions always return 0 or 1 (Uint8). - -The following types can be compared: - * numbers - * strings and fixed strings - * dates - * dates with times - -within each group, but not between different groups. - -For example, you can't compare a date with a string. You have to use a function to convert the string to a date, or vice versa. - -Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character. - -Note: before version 1.1.54134 signed and unsigned numbers were compared the same way as in C++. That is, you could got an incorrect result in such cases: SELECT 9223372036854775807 > -1. From version 1.1.54134, the behavior has changed and numbers are compared mathematically correct. - -equals, a = b and a == b operator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -notEquals, a != b and a <> b operator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -less, < operator -~~~~~~~~~~~~~~~~ - -greater, > operator -~~~~~~~~~~~~~~~~~~~ - -lessOrEquals, <= operator -~~~~~~~~~~~~~~~~~~~~~~~~~ - -greaterOrEquals, >= operator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/en/functions/conditional_functions.md b/docs/en/functions/conditional_functions.md new file mode 100644 index 00000000000..c658b35bcbc --- /dev/null +++ b/docs/en/functions/conditional_functions.md @@ -0,0 +1,6 @@ +# Conditional functions + +## if(cond, then, else), cond ? operator then : else + +Returns 'then' if cond !or 'else' if cond = 0.'cond' must be UInt 8, and 'then' and 'else' must be a type that has the smallest common type. + diff --git a/docs/en/functions/conditional_functions.rst b/docs/en/functions/conditional_functions.rst deleted file mode 100644 index 7b604ec08f8..00000000000 --- a/docs/en/functions/conditional_functions.rst +++ /dev/null @@ -1,7 +0,0 @@ -Conditional functions ---------------------- - -if(cond, then, else), ternary operator cond ? then : else -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns 'then' if 'cond != 0', or 'else' if 'cond = 0'. -'cond' must be UInt 8, and 'then' and 'else' must be a type that has the smallest common type. diff --git a/docs/en/functions/date_time_functions.rst b/docs/en/functions/date_time_functions.md similarity index 53% rename from docs/en/functions/date_time_functions.rst rename to docs/en/functions/date_time_functions.md index 0e9e5d91b44..162b09b8c16 100644 --- a/docs/en/functions/date_time_functions.rst +++ b/docs/en/functions/date_time_functions.md @@ -1,150 +1,145 @@ -Functions for working with dates and times ------------------------------------------- +# Functions for working with dates and times -Time Zone Support +Support for time zones -All functions for working with the date and time for which this makes sense, can take a second, optional argument - the time zone name. Example: Asia / Yekaterinburg. In this case, they do not use the local time zone (the default), but the specified one. +All functions for working with the date and time that have a logical use for the time zone can accept a second optional time zone argument. Example: Asia/Yekaterinburg. In this case, they use the specified time zone instead of the local (default) one. -.. code-block:: sql +```sql +SELECT + toDateTime('2016-06-15 23:00:00') AS time, + toDate(time) AS date_local, + toDate(time, 'Asia/Yekaterinburg') AS date_yekat, + toString(time, 'US/Samoa') AS time_samoa +``` - SELECT - toDateTime('2016-06-15 23:00:00') AS time, - toDate(time) AS date_local, - toDate(time, 'Asia/Yekaterinburg') AS date_yekat, - toString(time, 'US/Samoa') AS time_samoa +```text +┌────────────────time─┬─date_local─┬─date_yekat─┬─time_samoa──────────┐ +│ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-16 │ 2016-06-15 09:00:00 │ +└─────────────────────┴────────────┴────────────┴─────────────────────┘ +``` -.. code-block:: text +Only time zones that differ from UTC by a whole number of hours are supported. - ┌────────────────time─┬─date_local─┬─date_yekat─┬─time_samoa──────────┐ - │ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-16 │ 2016-06-15 09:00:00 │ - └─────────────────────┴────────────┴────────────┴─────────────────────┘ +## toYear -Only time zones are supported, different from UTC for an integer number of hours. - -toYear -~~~~~~ Converts a date or date with time to a UInt16 number containing the year number (AD). -toMonth -~~~~~~~ +## toMonth + Converts a date or date with time to a UInt8 number containing the month number (1-12). -toDayOfMonth -~~~~~~~~~~~~ -Converts a date or date with time to a UInt8 number containing the number of the day of the month (1-31). +## toDayOfMonth + +-Converts a date or date with time to a UInt8 number containing the number of the day of the month (1-31). + +## toDayOfWeek -toDayOfWeek -~~~~~~~~~~~ Converts a date or date with time to a UInt8 number containing the number of the day of the week (Monday is 1, and Sunday is 7). -toHour -~~~~~~ -Converts a date with time to a UInt8 number containing the number of the hour in 24-hour time (0-23). -This function assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true - even in Moscow the clocks were once changed at a different time). +## toHour + +Converts a date with time to a UInt8 number containing the number of the hour in 24-hour time (0-23). +This function assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time). + +## toMinute -toMinute -~~~~~~~~ Converts a date with time to a UInt8 number containing the number of the minute of the hour (0-59). -toSecond -~~~~~~~~ +## toSecond + Converts a date with time to a UInt8 number containing the number of the second in the minute (0-59). Leap seconds are not accounted for. -toStartOfDay -~~~~~~~~~~~~ -Rounds down a date with time to the start of the day. +## toMonday -toMonday -~~~~~~~~ Rounds down a date or date with time to the nearest Monday. Returns the date. -toStartOfMonth -~~~~~~~~~~~~~~ +## toStartOfMonth + Rounds down a date or date with time to the first day of the month. Returns the date. -toStartOfQuarter -~~~~~~~~~~~~~~~~ -Rounds down a date or date with time to the first day of the quarter. -The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October. Returns the date. +## toStartOfQuarter + +Rounds down a date or date with time to the first day of the quarter. +The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October. +Returns the date. + +## toStartOfYear -toStartOfYear -~~~~~~~~~~~~~ Rounds down a date or date with time to the first day of the year. Returns the date. -toStartOfMinute -~~~~~~~~~~~~~~~ +## toStartOfMinute + Rounds down a date with time to the start of the minute. -toStartOfFiveMinute -~~~~~~~~~~~~~~~~~~~ -Rounds down a date with time to the start of the 5 minute (00:00, 00:05, 00:10...). +## toStartOfFiveMinute -toStartOfFifteenMinutes -~~~~~~~~~~~~~~~~~~~ -Rounds down a date with time to the start of the 15 minutes (00:00, 00:15, 00:30...). - -toStartOfHour -~~~~~~~~~~~~~ Rounds down a date with time to the start of the hour. -toTime -~~~~~~ -Converts a date with time to some fixed date, while preserving the time. +Note: If you need to round a date with time to any other number of seconds, minutes, or hours, you can convert it into a number by using the toUInt32 function, then round the number using intDiv and multiplication, and convert it back using the toDateTime function. + +## toStartOfHour + +Rounds down a date with time to the start of the hour. + +## toTime + +Converts a date with time to a certain fixed date, while preserving the time. + +## toRelativeYearNum -toRelativeYearNum -~~~~~~~~~~~~~~~~~ Converts a date with time or date to the number of the year, starting from a certain fixed point in the past. -toRelativeMonthNum -~~~~~~~~~~~~~~~~~~ +## toRelativeMonthNum + Converts a date with time or date to the number of the month, starting from a certain fixed point in the past. -toRelativeWeekNum -~~~~~~~~~~~~~~~~~ +## toRelativeWeekNum + Converts a date with time or date to the number of the week, starting from a certain fixed point in the past. -toRelativeDayNum -~~~~~~~~~~~~~~~~ +## toRelativeDayNum + Converts a date with time or date to the number of the day, starting from a certain fixed point in the past. -toRelativeHourNum -~~~~~~~~~~~~~~~~~ +## toRelativeHourNum + Converts a date with time or date to the number of the hour, starting from a certain fixed point in the past. -toRelativeMinuteNum -~~~~~~~~~~~~~~~~~~~ +## toRelativeMinuteNum + Converts a date with time or date to the number of the minute, starting from a certain fixed point in the past. -toRelativeSecondNum -~~~~~~~~~~~~~~~~~~~ +## toRelativeSecondNum + Converts a date with time or date to the number of the second, starting from a certain fixed point in the past. -now -~~~ +## now + Accepts zero arguments and returns the current time at one of the moments of request execution. This function returns a constant, even if the request took a long time to complete. -today -~~~~~ +## today + Accepts zero arguments and returns the current date at one of the moments of request execution. The same as 'toDate(now())'. -yesterday -~~~~~~~~~ +## yesterday + Accepts zero arguments and returns yesterday's date at one of the moments of request execution. The same as 'today() - 1'. -timeSlot -~~~~~~~~ +## timeSlot + Rounds the time to the half hour. This function is specific to Yandex.Metrica, since half an hour is the minimum amount of time for breaking a session into two sessions if a counter shows a single user's consecutive pageviews that differ in time by strictly more than this amount. This means that tuples (the counter number, user ID, and time slot) can be used to search for pageviews that are included in the corresponding session. -timeSlots(StartTime, Duration) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## timeSlots(StartTime, Duration) + For a time interval starting at 'StartTime' and continuing for 'Duration' seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the half hour. -For example, ``timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]``. -This is necessary for searching for page views in the corresponding session. +For example, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. +This is necessary for searching for pageviews in the corresponding session. + diff --git a/docs/en/functions/encoding_functions.md b/docs/en/functions/encoding_functions.md new file mode 100644 index 00000000000..27ba1554a15 --- /dev/null +++ b/docs/en/functions/encoding_functions.md @@ -0,0 +1,27 @@ +# Encoding functions + +## hex + +Accepts arguments of types: `String`, `unsigned integer`, `Date`, or `DateTime`. Returns a string containing the argument's hexadecimal representation. Uses uppercase letters `A-F`. Does not use `0x` prefixes or `h` suffixes. For strings, all bytes are simply encoded as two hexadecimal numbers. Numbers are converted to big endian ("human readable") format. For numbers, older zeros are trimmed, but only by entire bytes. For example, `hex (1) = '01'`. `Date` is encoded as the number of days since the beginning of the Unix epoch. `DateTime` is encoded as the number of seconds since the beginning of the Unix epoch. + +## unhex(str) + +Accepts a string containing any number of hexadecimal digits, and returns a string containing the corresponding bytes. Supports both uppercase and lowercase letters A-F. The number of hexadecimal digits does not have to be even. If it is odd, the last digit is interpreted as the younger half of the 00-0F byte. If the argument string contains anything other than hexadecimal digits, some implementation-defined result is returned (an exception isn't thrown). +If you want to convert the result to a number, you can use the 'reverse' and 'reinterpretAsType' functions. + +## UUIDStringToNum(str) + +Accepts a string containing 36 characters in the format `123e4567-e89b-12d3-a456-426655440000`, and returns it as a set of bytes in a FixedString(16). + +## UUIDNumToString(str) + +Accepts a FixedString(16) value. Returns a string containing 36 characters in text format. + +## bitmaskToList(num) + +Accepts an integer. Returns a string containing the list of powers of two that total the source number when summed. They are comma-separated without spaces in text format, in ascending order. + +## bitmaskToArray(num) + +Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order. + diff --git a/docs/en/functions/encoding_functions.rst b/docs/en/functions/encoding_functions.rst deleted file mode 100644 index 0d7f9176771..00000000000 --- a/docs/en/functions/encoding_functions.rst +++ /dev/null @@ -1,31 +0,0 @@ -Encoding functions ------------------- - -hex -~~~ -Accepts a String, unsigned integer, Date, or DateTime. Returns a string containing the argument's hexadecimal representation. Uses uppercase letters A-F. -Doesn't use ``0x`` prefixes or ``h`` suffixes. -For strings, all bytes are simply encoded as two hexadecimal numbers. Numbers are converted to big endian ("human readable") format. -For numbers, older zeros are trimmed, but only by entire bytes. -For example, ``hex(1) = '01'``. Dates are encoded as the number of days since the beginning of the Unix Epoch. Dates with times are encoded as the number of seconds since the beginning of the Unix Epoch. - -unhex(str) -~~~~~~~~~~ -Accepts a string containing any number of hexadecimal digits, and returns a string containing the corresponding bytes. Supports both uppercase and lowercase letters A-F. The number of hexadecimal digits doesn't have to be even. If it is odd, the last digit is interpreted as the younger half of the 00-0F byte. If the argument string contains anything other than hexadecimal digits, some implementation-defined result is returned (an exception isn't thrown). -If you want to convert the result to a number, you can use the functions 'reverse' and 'reinterpretAsType' - -UUIDStringToNum(str) -~~~~~~~~~~~~~~~~~~~~ -Accepts a string containing the UUID in the text format (``123e4567-e89b-12d3-a456-426655440000``). Returns a binary representation of the UUID in ``FixedString(16)``. - -UUIDNumToString(str) -~~~~~~~~~~~~~~~~~~~~ -Accepts a FixedString(16) value containing the UUID in the binary format. Returns a readable string containing the UUID in the text format. - -bitmaskToList(num) -~~~~~~~~~~~~~~~~~~ -Accepts an integer. Returns a string containing the list of powers of two that total the source number when summed. They are comma-separated without spaces in text format, in ascending order. - -bitmaskToArray(num) -~~~~~~~~~~~~~~~~~~~ -Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order. diff --git a/docs/en/functions/ext_dict_functions.md b/docs/en/functions/ext_dict_functions.md new file mode 100644 index 00000000000..806b6a55d37 --- /dev/null +++ b/docs/en/functions/ext_dict_functions.md @@ -0,0 +1,46 @@ + + +# Functions for working with external dictionaries + +For information on connecting and configuring external dictionaries, see "[External dictionaries](../dicts/external_dicts.md#dicts-external_dicts)". + +## dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 + +## dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 + +## dictGetFloat32, dictGetFloat64 + +## dictGetDate, dictGetDateTime + +## dictGetUUID + +## dictGetString + +`dictGetT('dict_name', 'attr_name', id)` + +- Get the value of the attr_name attribute from the dict_name dictionary using the 'id' key.`dict_name` and `attr_name` are constant strings.`id`must be UInt64. +If there is no `id` key in the dictionary, it returns the default value specified in the dictionary description. + +## dictGetTOrDefault + +`dictGetT('dict_name', 'attr_name', id, default)` +Аналогично функциям dictGetT, но значение по умолчанию берётся из последнего аргумента функции. + +## dictIsIn + +`dictIsIn ('dict_name', child_id, ancestor_id)` + +- For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8. + +## dictGetHierarchy + +`dictGetHierarchy('dict_name', id)` + +- For the 'dict_name' hierarchical dictionary, returns an array of dictionary keys starting from 'id' and continuing along the chain of parent elements. Returns Array(UInt64). + +## dictHas + +`dictHas('dict_name', id)` + +- Check whether the dictionary has the key. Returns a UInt8 value equal to 0 if there is no key and 1 if there is a key. + diff --git a/docs/en/functions/ext_dict_functions.rst b/docs/en/functions/ext_dict_functions.rst deleted file mode 100644 index 7e2bf2b53d4..00000000000 --- a/docs/en/functions/ext_dict_functions.rst +++ /dev/null @@ -1,46 +0,0 @@ -Functions for working with external dictionaries ------------------------------------------------- -For more information, see the section "External dictionaries". - -dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -dictGetFloat32, dictGetFloat64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -dictGetDate, dictGetDateTime -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -dictGetUUID -~~~~~~~~~~~ - -dictGetString -~~~~~~~~~~~~~ -``dictGetT('dict_name', 'attr_name', id)`` -Gets the value of the 'attr_name' attribute from the 'dict_name' dictionary by the 'id' key. -'dict_name' and 'attr_name' are constant strings. -'id' must be UInt64. -If the 'id' key is not in the dictionary, it returns the default value set in the dictionary definition. - -dictGetTOrDefault -~~~~~~~~~~~~~~~~~ -``dictGetT('dict_name', 'attr_name', id, default)`` -Similar to the functions dictGetT, but the default value is taken from the last argument of the function. - -dictIsIn -~~~~~~~~ -``dictIsIn('dict_name', child_id, ancestor_id)`` -For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8. - -dictGetHierarchy -~~~~~~~~~~~~~~~~ -``dictGetHierarchy('dict_name', id)`` -For the 'dict_name' hierarchical dictionary, returns an array of dictionary keys starting from 'id' and continuing along the chain of parent elements. Returns Array(UInt64). - -dictHas -~~~~~~~ -``dictHas('dict_name', id)`` -check the presence of a key in the dictionary. Returns a value of type UInt8, equal to 0, if there is no key and 1 if there is a key. diff --git a/docs/en/functions/hash_functions.rst b/docs/en/functions/hash_functions.md similarity index 76% rename from docs/en/functions/hash_functions.rst rename to docs/en/functions/hash_functions.md index 24117576e38..42107ce5999 100644 --- a/docs/en/functions/hash_functions.rst +++ b/docs/en/functions/hash_functions.md @@ -1,69 +1,66 @@ -Hash functions --------------- +# Hash functions + Hash functions can be used for deterministic pseudo-random shuffling of elements. +## halfMD5 -halfMD5 -~~~~~~~ Calculates the MD5 from a string. Then it takes the first 8 bytes of the hash and interprets them as UInt64 in big endian. Accepts a String-type argument. Returns UInt64. This function works fairly slowly (5 million short strings per second per processor core). If you don't need MD5 in particular, use the 'sipHash64' function instead. -MD5 -~~~ +## MD5 + Calculates the MD5 from a string and returns the resulting set of bytes as FixedString(16). If you don't need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the 'sipHash128' function instead. -If you need the same result as gives 'md5sum' utility, write ``lower(hex(MD5(s)))``. +If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))). + +## sipHash64 -sipHash64 -~~~~~~~~~ Calculates SipHash from a string. Accepts a String-type argument. Returns UInt64. -SipHash is a cryptographic hash function. It works at least three times faster than MD5. For more information, see https://131002.net/siphash/ +SipHash is a cryptographic hash function. It works at least three times faster than MD5. +For more information, see the link: + +## sipHash128 -sipHash128 -~~~~~~~~~~ Calculates SipHash from a string. Accepts a String-type argument. Returns FixedString(16). -Differs from sipHash64 in that the final xor-folding state is only done up to 128 bits. +Differs from sipHash64 in that the final xor-folding state is only done up to 128 bytes. + +## cityHash64 -cityHash64 -~~~~~~~~~~ Calculates CityHash64 from a string or a similar hash function for any number of any type of arguments. For String-type arguments, CityHash is used. This is a fast non-cryptographic hash function for strings with decent quality. For other types of arguments, a decent implementation-specific fast non-cryptographic hash function is used. If multiple arguments are passed, the function is calculated using the same rules and chain combinations using the CityHash combinator. -For example, you can compute the checksum of an entire table with accuracy up to the row order: ``SELECT sum(cityHash64(*)) FROM table``. +For example, you can compute the checksum of an entire table with accuracy up to the row order: `SELECT sum(cityHash64(*)) FROM table`. + +## intHash32 -intHash32 -~~~~~~~~~ Calculates a 32-bit hash code from any type of integer. This is a relatively fast non-cryptographic hash function of average quality for numbers. -intHash64 -~~~~~~~~~ +## intHash64 + Calculates a 64-bit hash code from any type of integer. It works faster than intHash32. Average quality. -SHA1 -~~~~ +## SHA1 -SHA224 -~~~~~~ +## SHA224 + +## SHA256 -SHA256 -~~~~~~ Calculates SHA-1, SHA-224, or SHA-256 from a string and returns the resulting set of bytes as FixedString(20), FixedString(28), or FixedString(32). The function works fairly slowly (SHA-1 processes about 5 million short strings per second per processor core, while SHA-224 and SHA-256 process about 2.2 million). We recommend using this function only in cases when you need a specific hash function and you can't select it. Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in SELECTS. -URLHash(url[, N]) -~~~~~~~~~~~~~~~~~ +## URLHash(url\[, N\]) + A fast, decent-quality non-cryptographic hash function for a string obtained from a URL using some type of normalization. - -``URLHash(s)`` - Calculates a hash from a string without one of the trailing symbols ``/``,``?`` or ``#`` at the end, if present. - -``URL Hash(s, N)`` - Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols ``/``,``?`` or ``#`` at the end, if present. +`URLHash(s)` – Calculates a hash from a string without one of the trailing symbols `/`,`?` or `#` at the end, if present. +`URLHash(s, N)` – Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols `/`,`?` or `#` at the end, if present. Levels are the same as in URLHierarchy. This function is specific to Yandex.Metrica. + diff --git a/docs/en/functions/higher_order_functions.md b/docs/en/functions/higher_order_functions.md new file mode 100644 index 00000000000..912d0c3f388 --- /dev/null +++ b/docs/en/functions/higher_order_functions.md @@ -0,0 +1,73 @@ +# Higher-order functions + +## `->` operator, lambda(params, expr) function + +Allows describing a lambda function for passing to a higher-order function. The left side of the arrow has a formal parameter, which is any ID, or multiple formal parameters – any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns. + +Examples: `x -> 2 * x, str -> str != Referer.` + +Higher-order functions can only accept lambda functions as their functional argument. + +A lambda function that accepts multiple arguments can be passed to a higher-order function. In this case, the higher-order function is passed several arrays of identical length that these arguments will correspond to. + +For all functions other than 'arrayMap' and 'arrayFilter', the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed. + +### arrayMap(func, arr1, ...) + +Returns an array obtained from the original application of the 'func' function to each element in the 'arr' array. + +### arrayFilter(func, arr1, ...) + +Returns an array containing only the elements in 'arr1' for which 'func' returns something other than 0. + +Examples: + +```sql +SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res +``` + +```text +┌─res───────────┐ +│ ['abc World'] │ +└───────────────┘ +``` + +```sql +SELECT + arrayFilter( + (i, x) -> x LIKE '%World%', + arrayEnumerate(arr), + ['Hello', 'abc World'] AS arr) + AS res +``` + +```text +┌─res─┐ +│ [2] │ +└─────┘ +``` + +### arrayCount(\[func,\] arr1, ...) + +Returns the number of elements in the arr array for which func returns something other than 0. If 'func' is not specified, it returns the number of non-zero elements in the array. + +### arrayExists(\[func,\] arr1, ...) + +Returns 1 if there is at least one element in 'arr' for which 'func' returns something other than 0. Otherwise, it returns 0. + +### arrayAll(\[func,\] arr1, ...) + +Returns 1 if 'func' returns something other than 0 for all the elements in 'arr'. Otherwise, it returns 0. + +### arraySum(\[func,\] arr1, ...) + +Returns the sum of the 'func' values. If the function is omitted, it just returns the sum of the array elements. + +### arrayFirst(func, arr1, ...) + +Returns the first element in the 'arr1' array for which 'func' returns something other than 0. + +### arrayFirstIndex(func, arr1, ...) + +Returns the index of the first element in the 'arr1' array for which 'func' returns something other than 0. + diff --git a/docs/en/functions/higher_order_functions.rst b/docs/en/functions/higher_order_functions.rst deleted file mode 100644 index b37741919f2..00000000000 --- a/docs/en/functions/higher_order_functions.rst +++ /dev/null @@ -1,73 +0,0 @@ -Higher-order functions ----------------------- - --> operator, lambda(params, expr) function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Allows describing a lambda function for passing to a higher-order function. The left side of the arrow has a formal parameter - any ID, or multiple formal parameters - any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns. - -Examples: ``x -> 2 * x, str -> str != Referer.`` - -Higher-order functions can only accept lambda functions as their functional argument. - -A lambda function that accepts multiple arguments can be passed to a higher-order function. In this case, the higher-order function is passed several arrays of identical length that these arguments will correspond to. - -For all functions other than 'arrayMap' and 'arrayFilter', the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed. - -arrayMap(func, arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns an array obtained from the original application of the 'func' function to each element in the 'arr' array. - -arrayFilter(func, arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns an array containing only the elements in 'arr1' for which 'func' returns something other than 0. - -Examples: - -.. code-block:: sql - - SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res - -.. code-block:: text - - ┌─res───────────┐ - │ ['abc World'] │ - └───────────────┘ - -.. code-block:: sql - - SELECT - arrayFilter( - (i, x) -> x LIKE '%World%', - arrayEnumerate(arr), - ['Hello', 'abc World'] AS arr) - AS res - -.. code-block:: text - - ┌─res─┐ - │ [2] │ - └─────┘ - -arrayCount([func,] arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns the number of elements in 'arr' for which 'func' returns something other than 0. If 'func' is not specified, it returns the number of non-zero items in the array. - -arrayExists([func,] arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns 1 if there is at least one element in 'arr' for which 'func' returns something other than 0. Otherwise, it returns 0. - -arrayAll([func,] arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns 1 if 'func' returns something other than 0 for all the elements in 'arr'. Otherwise, it returns 0. - -arraySum([func,] arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns the sum of the 'func' values. If the function is omitted, it just returns the sum of the array elements. - -arrayFirst(func, arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns the first element in the 'arr1' array for which 'func' returns something other than 0. - -arrayFirstIndex(func, arr1, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns the index of the first element in the 'arr1' array for which 'func' returns something other than 0. diff --git a/docs/en/functions/in_functions.md b/docs/en/functions/in_functions.md new file mode 100644 index 00000000000..27ac483cdb5 --- /dev/null +++ b/docs/en/functions/in_functions.md @@ -0,0 +1,18 @@ +# Functions for implementing the IN operator + +## in, notIn, globalIn, globalNotIn + +See the section "IN operators". + +## tuple(x, y, ...), operator (x, y, ...) + +A function that allows grouping multiple columns. +For columns with the types T1, T2, ..., it returns a Tuple(T1, T2, ...) type tuple containing these columns. There is no cost to execute the function. +Tuples are normally used as intermediate values for an argument of IN operators, or for creating a list of formal parameters of lambda functions. Tuples can't be written to a table. + +## tupleElement(tuple, n), operator x.N + +A function that allows getting a column from a tuple. +'N' is the column index, starting from 1. N must be a constant. 'N' must be a constant. 'N' must be a strict postive integer no greater than the size of the tuple. +There is no cost to execute the function. + diff --git a/docs/en/functions/in_functions.rst b/docs/en/functions/in_functions.rst deleted file mode 100644 index dbd4f7893eb..00000000000 --- a/docs/en/functions/in_functions.rst +++ /dev/null @@ -1,18 +0,0 @@ -Functions for implementing the IN operator ------------------------------------------- - -in, notIn, globalIn, globalNotIn -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -See the section "IN operators". - -tuple(x, y, ...), operator (x, y, ...) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A function that allows grouping multiple columns. -For columns with the types T1, T2, ..., it returns a Tuple(T1, T2, ...) type tuple containing these columns. There is no cost to execute the function. -Tuples are normally used as intermediate values for an argument of IN operators, or for creating a list of formal parameters of lambda functions. Tuples can't be written to a table. - -tupleElement(tuple, n), operator x.N -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A function that allows getting columns from a tuple. -'N' is the column index, starting from 1. 'N' must be a constant. 'N' must be a strict postive integer no greater than the size of the tuple. -There is no cost to execute the function. diff --git a/docs/en/functions/index.rst b/docs/en/functions/index.md similarity index 71% rename from docs/en/functions/index.rst rename to docs/en/functions/index.md index 2d9b7a2ec4d..5250025757a 100644 --- a/docs/en/functions/index.rst +++ b/docs/en/functions/index.md @@ -1,33 +1,32 @@ -Functions -========= +# Functions There are at least\* two types of functions - regular functions (they are just called "functions") and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn't depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). In this section we discuss regular functions. For aggregate functions, see the section "Aggregate functions". -\* - there is a third type of function that the 'arrayJoin' function belongs to; table functions can also be mentioned separately. - +\* - There is a third type of function that the 'arrayJoin' function belongs to; table functions can also be mentioned separately.\* +```eval_rst .. toctree:: :glob: * +``` +## Strong typing -Strong typing -~~~~~~~~~~~~~ In contrast to standard SQL, ClickHouse has strong typing. In other words, it doesn't make implicit conversions between types. Each function works for a specific set of types. This means that sometimes you need to use type conversion functions. -Common subexpression elimination -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Сommon subexpression elimination + All expressions in a query that have the same AST (the same record or same result of syntactic parsing) are considered to have identical values. Such expressions are concatenated and executed once. Identical subqueries are also eliminated this way. -Types of results -~~~~~~~~~~~~~~~~ +## Types of results + All functions return a single return as the result (not several values, and not zero values). The type of result is usually defined only by the types of arguments, not by the values. Exceptions are the tupleElement function (the a.N operator), and the toFixedString function. -Constants -~~~~~~~~~ +## Constants + For simplicity, certain functions can only work with constants for some arguments. For example, the right argument of the LIKE operator must be a constant. Almost all functions return a constant for constant arguments. The exception is functions that generate random numbers. The 'now' function returns different values for queries that were run at different times, but the result is considered a constant, since constancy is only important within a single query. @@ -35,36 +34,32 @@ A constant expression is also considered a constant (for example, the right half Functions can be implemented in different ways for constant and non-constant arguments (different code is executed). But the results for a constant and for a true column containing only the same value should match each other. -Immutability -~~~~~~~~~~~~ +## Constancy -Functions can't change the values of their arguments - any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. +Functions can't change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query. - -Error handling -~~~~~~~~~~~~~~ +## Error handling Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query. +## Evaluation of argument expressions -Evaluation of argument expressions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In almost all programming languages, one of the arguments might not be evaluated for certain operators. This is usually for the operators ``&&``, ``||``, ``?:``. +In almost all programming languages, one of the arguments might not be evaluated for certain operators. This is usually the operators `&&`, `||`, and `?:`. But in ClickHouse, arguments of functions (operators) are always evaluated. This is because entire parts of columns are evaluated at once, instead of calculating each row separately. -Performing functions for distributed query processing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Performing functions for distributed query processing For distributed query processing, as many stages of query processing as possible are performed on remote servers, and the rest of the stages (merging intermediate results and everything after that) are performed on the requestor server. This means that functions can be performed on different servers. -For example, in the query ``SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y)``, -- if distributed_table has at least two shards, the functions ``g`` and ``h`` are performed on remote servers, and the function ``f`` - is performed on the requestor server. -- if distributed_table has only one shard, all the functions ``f``, ``g``, and ``h`` are performed on this shard's server. +For example, in the query `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h(y),` + +- if a `distributed_table` has at least two shards, the functions 'g' and 'h' are performed on remote servers, and the function 'f' is performed on the requestor server. +- if a `distributed_table` has only one shard, all the 'f', 'g', and 'h' functions are performed on this shard's server. The result of a function usually doesn't depend on which server it is performed on. However, sometimes this is important. For example, functions that work with dictionaries use the dictionary that exists on the server they are running on. -Another example is the hostName function, which returns the name of the server it is running on in order to make GROUP BY by servers in a SELECT query. +Another example is the `hostName` function, which returns the name of the server it is running on in order to make `GROUP BY` by servers in a `SELECT` query. + +If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an 'any' aggregate function or add it to a key in `GROUP BY`. -If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an 'any' aggregate function or add it to a key in GROUP BY. diff --git a/docs/en/functions/ip_address_functions.md b/docs/en/functions/ip_address_functions.md new file mode 100644 index 00000000000..f83f75319c7 --- /dev/null +++ b/docs/en/functions/ip_address_functions.md @@ -0,0 +1,115 @@ +# Functions for working with IP addresses + +## IPv4NumToString(num) + +Takes a UInt32 number. Interprets it as an IPv4 address in big endian. Returns a string containing the corresponding IPv4 address in the format A.B.C.d (dot-separated numbers in decimal form). + +## IPv4StringToNum(s) + +The reverse function of IPv4NumToString. If the IPv4 address has an invalid format, it returns 0. + +## IPv4NumToStringClassC(num) + +Similar to IPv4NumToString, but using xxx instead of the last octet. + +Example: + +```sql +SELECT + IPv4NumToStringClassC(ClientIP) AS k, + count() AS c +FROM test.hits +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +```text +┌─k──────────────┬─────c─┐ +│ 83.149.9.xxx │ 26238 │ +│ 217.118.81.xxx │ 26074 │ +│ 213.87.129.xxx │ 25481 │ +│ 83.149.8.xxx │ 24984 │ +│ 217.118.83.xxx │ 22797 │ +│ 78.25.120.xxx │ 22354 │ +│ 213.87.131.xxx │ 21285 │ +│ 78.25.121.xxx │ 20887 │ +│ 188.162.65.xxx │ 19694 │ +│ 83.149.48.xxx │ 17406 │ +└────────────────┴───────┘ +``` + +Since using 'xxx' is highly unusual, this may be changed in the future. We recommend that you don't rely on the exact format of this fragment. + +### IPv6NumToString(x) + +Accepts a FixedString(16) value containing the IPv6 address in binary format. Returns a string containing this address in text format. +IPv6-mapped IPv4 addresses are output in the format ::ffff:111.222.33.44. Examples: + +```sql +SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr +``` + +```text +┌─addr─────────┐ +│ 2a02:6b8::11 │ +└──────────────┘ +``` + +```sql +SELECT + IPv6NumToString(ClientIP6 AS k), + count() AS c +FROM hits_all +WHERE EventDate = today() AND substring(ClientIP6, 1, 12) != unhex('00000000000000000000FFFF') +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +```text +┌─IPv6NumToString(ClientIP6)──────────────┬─────c─┐ +│ 2a02:2168:aaa:bbbb::2 │ 24695 │ +│ 2a02:2698:abcd:abcd:abcd:abcd:8888:5555 │ 22408 │ +│ 2a02:6b8:0:fff::ff │ 16389 │ +│ 2a01:4f8:111:6666::2 │ 16016 │ +│ 2a02:2168:888:222::1 │ 15896 │ +│ 2a01:7e00::ffff:ffff:ffff:222 │ 14774 │ +│ 2a02:8109:eee:ee:eeee:eeee:eeee:eeee │ 14443 │ +│ 2a02:810b:8888:888:8888:8888:8888:8888 │ 14345 │ +│ 2a02:6b8:0:444:4444:4444:4444:4444 │ 14279 │ +│ 2a01:7e00::ffff:ffff:ffff:ffff │ 13880 │ +└─────────────────────────────────────────┴───────┘ +``` + +```sql +SELECT + IPv6NumToString(ClientIP6 AS k), + count() AS c +FROM hits_all +WHERE EventDate = today() +GROUP BY k +ORDER BY c DESC +LIMIT 10 +``` + +```text +┌─IPv6NumToString(ClientIP6)─┬──────c─┐ +│ ::ffff:94.26.111.111 │ 747440 │ +│ ::ffff:37.143.222.4 │ 529483 │ +│ ::ffff:5.166.111.99 │ 317707 │ +│ ::ffff:46.38.11.77 │ 263086 │ +│ ::ffff:79.105.111.111 │ 186611 │ +│ ::ffff:93.92.111.88 │ 176773 │ +│ ::ffff:84.53.111.33 │ 158709 │ +│ ::ffff:217.118.11.22 │ 154004 │ +│ ::ffff:217.118.11.33 │ 148449 │ +│ ::ffff:217.118.11.44 │ 148243 │ +└────────────────────────────┴────────┘ +``` + +## IPv6StringToNum(s) + +The reverse function of IPv6NumToString. If the IPv6 address has an invalid format, it returns a string of null bytes. +HEX can be uppercase or lowercase. + diff --git a/docs/en/functions/ip_address_functions.rst b/docs/en/functions/ip_address_functions.rst deleted file mode 100644 index 9220092a5ff..00000000000 --- a/docs/en/functions/ip_address_functions.rst +++ /dev/null @@ -1,105 +0,0 @@ -Functions for working with IP addresses ---------------------------------------- - -IPv4NumToString(num) -~~~~~~~~~~~~~~~~~~~~ -Takes a UInt32 number. Interprets it as an IPv4 address in big endian. Returns a string containing the corresponding IPv4 address in the format A.B.C.d (dot-separated numbers in decimal form). - -IPv4StringToNum(s) -~~~~~~~~~~~~~~~~~~ -The reverse function of IPv4NumToString. If the IPv4 address has an invalid format, it returns 0. - -IPv4NumToStringClassC(num) -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Similar to IPv4NumToString, but using ``xxx`` instead of the last octet. - -Example: - -.. code-block:: sql - - SELECT - IPv4NumToStringClassC(ClientIP) AS k, - count() AS c - FROM test.hits - GROUP BY k - ORDER BY c DESC - LIMIT 10 - -.. code-block:: text - - ┌─k──────────────┬─────c─┐ - │ 83.149.9.xxx │ 26238 │ - │ 217.118.81.xxx │ 26074 │ - │ 213.87.129.xxx │ 25481 │ - │ 83.149.8.xxx │ 24984 │ - │ 217.118.83.xxx │ 22797 │ - │ 78.25.120.xxx │ 22354 │ - │ 213.87.131.xxx │ 21285 │ - │ 78.25.121.xxx │ 20887 │ - │ 188.162.65.xxx │ 19694 │ - │ 83.149.48.xxx │ 17406 │ - └────────────────┴───────┘ - -Since using ``'xxx'`` is highly unusual, this may be changed in the future. We recommend that you don't rely on the exact format of this fragment. - -IPv6NumToString(x) -~~~~~~~~~~~~~~~~~~ -Accepts a FixedString(16) value containing the IPv6 address in binary format. Returns a string containing this address in text format. -IPv6-mapped IPv4 addresses are output in the format ``::ffff:111.222.33.44``. Examples: - -.. code-block:: sql - - SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr - -.. code-block:: text - - ┌─addr─────────┐ - │ 2a02:6b8::11 │ - └──────────────┘ - SELECT - IPv6NumToString(ClientIP6 AS k), - count() AS c - FROM hits_all - WHERE EventDate = today() AND substring(ClientIP6, 1, 12) != unhex('00000000000000000000FFFF') - GROUP BY k - ORDER BY c DESC - LIMIT 10 - - ┌─IPv6NumToString(ClientIP6)──────────────┬─────c─┐ - │ 2a02:2168:aaa:bbbb::2 │ 24695 │ - │ 2a02:2698:abcd:abcd:abcd:abcd:8888:5555 │ 22408 │ - │ 2a02:6b8:0:fff::ff │ 16389 │ - │ 2a01:4f8:111:6666::2 │ 16016 │ - │ 2a02:2168:888:222::1 │ 15896 │ - │ 2a01:7e00::ffff:ffff:ffff:222 │ 14774 │ - │ 2a02:8109:eee:ee:eeee:eeee:eeee:eeee │ 14443 │ - │ 2a02:810b:8888:888:8888:8888:8888:8888 │ 14345 │ - │ 2a02:6b8:0:444:4444:4444:4444:4444 │ 14279 │ - │ 2a01:7e00::ffff:ffff:ffff:ffff │ 13880 │ - └─────────────────────────────────────────┴───────┘ - SELECT - IPv6NumToString(ClientIP6 AS k), - count() AS c - FROM hits_all - WHERE EventDate = today() - GROUP BY k - ORDER BY c DESC - LIMIT 10 - - ┌─IPv6NumToString(ClientIP6)─┬──────c─┐ - │ ::ffff:94.26.111.111 │ 747440 │ - │ ::ffff:37.143.222.4 │ 529483 │ - │ ::ffff:5.166.111.99 │ 317707 │ - │ ::ffff:46.38.11.77 │ 263086 │ - │ ::ffff:79.105.111.111 │ 186611 │ - │ ::ffff:93.92.111.88 │ 176773 │ - │ ::ffff:84.53.111.33 │ 158709 │ - │ ::ffff:217.118.11.22 │ 154004 │ - │ ::ffff:217.118.11.33 │ 148449 │ - │ ::ffff:217.118.11.44 │ 148243 │ - └────────────────────────────┴────────┘ - -IPv6StringToNum(s) -~~~~~~~~~~~~~~~~~~ -The reverse function of IPv6NumToString. If the IPv6 address has an invalid format, it returns a string of null bytes. -HEX can be uppercase or lowercase. diff --git a/docs/en/functions/json_functions.md b/docs/en/functions/json_functions.md new file mode 100644 index 00000000000..82b92272874 --- /dev/null +++ b/docs/en/functions/json_functions.md @@ -0,0 +1,54 @@ +# Functions for working with JSON. + +In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done. + +The following assumptions are made: + +1. The field name (function argument) must be a constant. +2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, но `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` +3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used. +4. The JSON doesn't have space characters outside of string literals. + +## visitParamHas(params, name) + +Checks whether there is a field with the 'name' name. + +## visitParamExtractUInt(params, name) + +Parses UInt64 from the value of the field named 'name'. If this is a string field, it tries to parse a number from the beginning of the string. If the field doesn't exist, or it exists but doesn't contain a number, it returns 0. + +## visitParamExtractInt(params, name) + +The same as for Int64. + +## visitParamExtractFloat(params, name) + +The same as for Float64. + +## visitParamExtractBool(params, name) + +Parses a true/false value. The result is UInt8. + +## visitParamExtractRaw(params, name) + +Returns the value of a field, including separators. + +Examples: + +```text +visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"' +visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}' +``` + +## visitParamExtractString(params, name) + +Parses the string in double quotes. The value is unescaped. If unescaping failed, it returns an empty string. + +Examples: + +```text +visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''visitParamExtractString('{"abc":"hello}', 'abc') = '' +``` + +There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8). + diff --git a/docs/en/functions/json_functions.rst b/docs/en/functions/json_functions.rst deleted file mode 100644 index 1a31ed726f4..00000000000 --- a/docs/en/functions/json_functions.rst +++ /dev/null @@ -1,55 +0,0 @@ -Functions for working with JSON. --------------------------------- -In Yandex.Metrica, JSON is passed by users as session parameters. There are several functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try not to do anything. - -The following assumptions are made: - #. The field name (function argument) must be a constant. - #. The field name is somehow canonically encoded in JSON. For example, ``visitParamHas('{"abc":"def"}', 'abc') = 1``, but ``visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0`` - #. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used. - #. JSON doesn't have space characters outside of string literals. - -visitParamHas(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Checks whether there is a field with the 'name' name. - -visitParamExtractUInt(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Parses UInt64 from the value of the field named 'name'. If this is a string field, it tries to parse a number from the beginning of the string. If the field doesn't exist, or it exists but doesn't contain a number, it returns 0. - -visitParamExtractInt(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The same as for Int64. - -visitParamExtractFloat(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The same as for Float64. - -visitParamExtractBool(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Parses a true/false value. The result is UInt8. - -visitParamExtractRaw(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns the value of a field, including separators. - -Examples: - -.. code-block:: text - - visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"' - visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}' - -visitParamExtractString(params, name) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Parses the string in double quotes. The value is unescaped. If unescaping failed, it returns an empty string. - -Examples: - -.. code-block:: text - - visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' - visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺' - visitParamExtractString('{"abc":"\\u263"}', 'abc') = '' - visitParamExtractString('{"abc":"hello}', 'abc') = '' - -Currently, there is no support for code points not from the basic multilingual plane written in the format ``\uXXXX\uYYYY`` (they are converted to CESU-8 instead of UTF-8). diff --git a/docs/en/functions/logical_functions.md b/docs/en/functions/logical_functions.md new file mode 100644 index 00000000000..4ef0fe5fd32 --- /dev/null +++ b/docs/en/functions/logical_functions.md @@ -0,0 +1,14 @@ +# Logical functions + +Logical functions accept any numeric types, but return a UInt8 number equal to 0 or 1. + +Zero as an argument is considered "false," while any non-zero value is considered "true". + +## and, AND operator + +## or, OR operator + +## not, NOT operator + +## xor + diff --git a/docs/en/functions/logical_functions.rst b/docs/en/functions/logical_functions.rst deleted file mode 100644 index 286ba8c4a21..00000000000 --- a/docs/en/functions/logical_functions.rst +++ /dev/null @@ -1,19 +0,0 @@ -Logical functions ------------------ - -Logical functions accept any numeric types, but return a UInt8 number equal to 0 or 1. - -Zero as an argument is considered "false," while any non-zero value is considered "true".. - -and, AND operator -~~~~~~~~~~~~~~~~~ - -or, OR operator -~~~~~~~~~~~~~~~ - -not, NOT operator -~~~~~~~~~~~~~~~~~ - -xor -~~~ - diff --git a/docs/en/functions/math_functions.rst b/docs/en/functions/math_functions.md similarity index 62% rename from docs/en/functions/math_functions.rst rename to docs/en/functions/math_functions.md index f7b722f8dba..d606c87a509 100644 --- a/docs/en/functions/math_functions.rst +++ b/docs/en/functions/math_functions.md @@ -1,100 +1,100 @@ -Mathematical functions ----------------------- +# Mathematical functions + All the functions return a Float64 number. The accuracy of the result is close to the maximum precision possible, but the result might not coincide with the machine representable number nearest to the corresponding real number. -e() -~~~ +## e() + Accepts zero arguments and returns a Float64 number close to the e number. -pi() -~~~~ +## pi() + Accepts zero arguments and returns a Float64 number close to π. -exp(x) -~~~~~~ +## exp(x) + Accepts a numeric argument and returns a Float64 number close to the exponent of the argument. -log(x) -~~~~~~ +## log(x) + Accepts a numeric argument and returns a Float64 number close to the natural logarithm of the argument. -exp2(x) -~~~~~~~ +## exp2(x) + Accepts a numeric argument and returns a Float64 number close to 2x. -log2(x) -~~~~~~~ +## log2(x) + Accepts a numeric argument and returns a Float64 number close to the binary logarithm of the argument. -exp10(x) -~~~~~~~~ +## exp10(x) + Accepts a numeric argument and returns a Float64 number close to 10x. -log10(x) -~~~~~~~~ +## log10(x) + Accepts a numeric argument and returns a Float64 number close to the decimal logarithm of the argument. -sqrt(x) -~~~~~~~ +## sqrt(x) + Accepts a numeric argument and returns a Float64 number close to the square root of the argument. -cbrt(x) -~~~~~~~ +## cbrt(x) + Accepts a numeric argument and returns a Float64 number close to the cubic root of the argument. -erf(x) -~~~~~~ +## erf(x) -If 'x' is non-negative, then ``erf(x / σ√2)`` - is the probability that a random variable having a normal distribution with standard deviation 'σ' takes the value that is separated from the expected value by more than 'x'. +If 'x' is non-negative, then erf(x / σ√2) is the probability that a random variable having a normal distribution with standard deviation 'σ' takes the value that is separated from the expected value by more than 'x'. Example (three sigma rule): -.. code-block:: sql +```sql +SELECT erf(3 / sqrt(2)) +``` - SELECT erf(3 / sqrt(2)) +```text +┌─erf(divide(3, sqrt(2)))─┐ +│ 0.9973002039367398 │ +└─────────────────────────┘ +``` -.. code-block:: text +## erfc(x) - ┌─erf(divide(3, sqrt(2)))─┐ - │ 0.9973002039367398 │ - └─────────────────────────┘ - -erfc(x) -~~~~~~~ Accepts a numeric argument and returns a Float64 number close to 1 - erf(x), but without loss of precision for large 'x' values. -lgamma(x) -~~~~~~~~~ +## lgamma(x) + The logarithm of the gamma function. -tgamma(x) -~~~~~~~~~ +## tgamma(x) + Gamma function. -sin(x) -~~~~~~ +## sin(x) + The sine. -cos(x) -~~~~~~ +## cos(x) + The cosine. -tan(x) -~~~~~~ +## tan(x) + The tangent. -asin(x) -~~~~~~~ -The arc sine +## asin(x) + +The arc sine. + +## acos(x) -acos(x) -~~~~~~~ The arc cosine. -atan(x) -~~~~~~~ +## atan(x) + The arc tangent. -pow(x, y) -~~~~~~~~~ -x to the power y. +## pow(x, y) + +xy. + diff --git a/docs/en/functions/other_functions.md b/docs/en/functions/other_functions.md new file mode 100644 index 00000000000..1cc627fd22e --- /dev/null +++ b/docs/en/functions/other_functions.md @@ -0,0 +1,281 @@ +# Other functions + +## hostName() + +Returns a string with the name of the host that this function was performed on. For distributed processing, this is the name of the remote server host, if the function is performed on a remote server. + +## visibleWidth(x) + +Calculates the approximate width when outputting values to the console in text format (tab-separated). +This function is used by the system for implementing Pretty formats. + +## toTypeName(x) + +Returns a string containing the type name of the passed argument. + +## blockSize() + +Gets the size of the block. +In ClickHouse, queries are always run on blocks (sets of column parts). This function allows getting the size of the block that you called it for. + +## materialize(x) + +Turns a constant into a full column containing just one value. +In ClickHouse, full columns and constants are represented differently in memory. Functions work differently for constant arguments and normal arguments (different code is executed), although the result is almost always the same. This function is for debugging this behavior. + +## ignore(...) + +Accepts any arguments and always returns 0. +However, the argument is still evaluated. This can be used for benchmarks. + +## sleep(seconds) + +Sleeps 'seconds' seconds on each data block. You can specify an integer or a floating-point number. + +## currentDatabase() + +Returns the name of the current database. +You can use this function in table engine parameters in a CREATE TABLE query where you need to specify the database. + +## isFinite(x) + +Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is not infinite and not a NaN, otherwise 0. + +## isInfinite(x) + +Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is infinite, otherwise 0. Note that 0 is returned for a NaN. + +## isNaN(x) + +Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is a NaN, otherwise 0. + +## hasColumnInTable(\['hostname'\[, 'username'\[, 'password'\]\],\] 'database', 'table', 'column') + +Accepts constant strings: database name, table name, and column name. Returns a UInt8 constant expression equal to 1 if there is a column, otherwise 0. If the hostname parameter is set, the test will run on a remote server. +The function throws an exception if the table does not exist. +For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. + +## bar + +Allows building a unicode-art diagram. + +`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction. + +The band is drawn with accuracy to one eighth of a symbol. + +Example: + +```sql +SELECT + toHour(EventTime) AS h, + count() AS c, + bar(c, 0, 600000, 20) AS bar +FROM test.hits +GROUP BY h +ORDER BY h ASC +``` + +```text +┌──h─┬──────c─┬─bar────────────────┐ +│ 0 │ 292907 │ █████████▋ │ +│ 1 │ 180563 │ ██████ │ +│ 2 │ 114861 │ ███▋ │ +│ 3 │ 85069 │ ██▋ │ +│ 4 │ 68543 │ ██▎ │ +│ 5 │ 78116 │ ██▌ │ +│ 6 │ 113474 │ ███▋ │ +│ 7 │ 170678 │ █████▋ │ +│ 8 │ 278380 │ █████████▎ │ +│ 9 │ 391053 │ █████████████ │ +│ 10 │ 457681 │ ███████████████▎ │ +│ 11 │ 493667 │ ████████████████▍ │ +│ 12 │ 509641 │ ████████████████▊ │ +│ 13 │ 522947 │ █████████████████▍ │ +│ 14 │ 539954 │ █████████████████▊ │ +│ 15 │ 528460 │ █████████████████▌ │ +│ 16 │ 539201 │ █████████████████▊ │ +│ 17 │ 523539 │ █████████████████▍ │ +│ 18 │ 506467 │ ████████████████▊ │ +│ 19 │ 520915 │ █████████████████▎ │ +│ 20 │ 521665 │ █████████████████▍ │ +│ 21 │ 542078 │ ██████████████████ │ +│ 22 │ 493642 │ ████████████████▍ │ +│ 23 │ 400397 │ █████████████▎ │ +└────┴────────┴────────────────────┘ +``` + + + +## transform + +Transforms a value according to the explicitly defined mapping of some elements to other ones. +There are two variations of this function: + +1. `transform(x, array_from, array_to, default)` + +`x` – What to transform. + +`array_from` – Constant array of values for converting. + +`array_to` – Constant array of values to convert the values in 'from' to. + +`default` – Which value to use if 'x' is not equal to any of the values in 'from'. + +`array_from` and `array_to` – Arrays of the same size. + +Types: + +`transform(T, Array(T), Array(U), U) -> U` + +`T` and `U` can be numeric, string, or Date or DateTime types. +Where the same letter is indicated (T or U), for numeric types these might not be matching types, but types that have a common type. +For example, the first argument can have the Int64 type, while the second has the Array(Uint16) type. + +If the 'x' value is equal to one of the elements in the 'array_from' array, it returns the existing element (that is numbered the same) from the 'array_to' array. Otherwise, it returns 'default'. If there are multiple matching elements in 'array_from', it returns one of the matches. + +Example: + +```sql +SELECT + transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other' AS title, + count() AS c +FROM test.hits +WHERE SearchEngineID != 0 +GROUP BY title +ORDER BY c DESC +``` + +```text +┌─title─────┬──────c─┐ +│ Yandex │ 498635 │ +│ Google │ 229872 │ +│ Other │ 104472 │ +└───────────┴────────┘ +``` + +2. `transform(x, array_from, array_to)` + +Differs from the first variation in that the 'default' argument is omitted. +If the 'x' value is equal to one of the elements in the 'array_from' array, it returns the matching element (that is numbered the same) from the 'array_to' array. Otherwise, it returns 'x'. + +Types: + +`transform(T, Array(T), Array(T)) -> T` + +Example: + +```sql +SELECT + transform(domain(Referer), ['yandex.ru', 'google.ru', 'vk.com'], ['www.yandex', 'ввв.яндекс.рф', 'example.com']) AS s, + count() AS c +FROM test.hits +GROUP BY domain(Referer) +ORDER BY count() DESC +LIMIT 10 +``` + +```text +┌─s──────────────┬───────c─┐ +│ │ 2906259 │ +│ www.yandex │ 867767 │ +│ ███████.ru │ 313599 │ +│ mail.yandex.ru │ 107147 │ +│ ввв.яндекс.рф │ 105668 │ +│ ██████.ru │ 100355 │ +│ █████████.ru │ 65040 │ +│ news.yandex.ru │ 64515 │ +│ ██████.net │ 59141 │ +│ example.com │ 57316 │ +└────────────────┴─────────┘ +``` + +## formatReadableSize(x) + +Accepts the size (number of bytes). Returns a rounded size with a suffix (KiB, MiB, etc.) as a string. + +Example: + +```sql +SELECT + arrayJoin([1, 1024, 1024*1024, 192851925]) AS filesize_bytes, + formatReadableSize(filesize_bytes) AS filesize +``` + +```text +┌─filesize_bytes─┬─filesize───┐ +│ 1 │ 1.00 B │ +│ 1024 │ 1.00 KiB │ +│ 1048576 │ 1.00 MiB │ +│ 192851925 │ 183.92 MiB │ +└────────────────┴────────────┘ +``` + +## least(a, b) + +Returns the smallest value from a and b. + +## greatest(a, b) + +Returns the largest value of a and b. + +## uptime() + +Returns the server's uptime in seconds. + +## version() + +Returns the version of the server as a string. + +## rowNumberInAllBlocks() + +Returns the ordinal number of the row in the data block. This function only considers the affected data blocks. + +## runningDifference(x) + +Calculates the difference between successive row values ​​in the data block. +Returns 0 for the first row and the difference from the previous row for each subsequent row. + +The result of the function depends on the affected data blocks and the order of data in the block. +If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +Example: + +```sql +SELECT + EventID, + EventTime, + runningDifference(EventTime) AS delta +FROM +( + SELECT + EventID, + EventTime + FROM events + WHERE EventDate = '2016-11-24' + ORDER BY EventTime ASC + LIMIT 5 +) +``` + +```text +┌─EventID─┬───────────EventTime─┬─delta─┐ +│ 1106 │ 2016-11-24 00:00:04 │ 0 │ +│ 1107 │ 2016-11-24 00:00:05 │ 1 │ +│ 1108 │ 2016-11-24 00:00:05 │ 0 │ +│ 1109 │ 2016-11-24 00:00:09 │ 4 │ +│ 1110 │ 2016-11-24 00:00:10 │ 1 │ +└─────────┴─────────────────────┴───────┘ +``` + +## MACNumToString(num) + +Accepts a UInt64 number. Interprets it as a MAC address in big endian. Returns a string containing the corresponding MAC address in the format AA:BB:CC:DD:EE:FF (colon-separated numbers in hexadecimal form). + +## MACStringToNum(s) + +The inverse function of MACNumToString. If the MAC address has an invalid format, it returns 0. + +## MACStringToOUI(s) + +Accepts a MAC address in the format AA:BB:CC:DD:EE:FF (colon-separated numbers in hexadecimal form). Returns the first three octets as a UInt64 number. If the MAC address has an invalid format, it returns 0. + diff --git a/docs/en/functions/other_functions.rst b/docs/en/functions/other_functions.rst deleted file mode 100644 index 861a3ac168f..00000000000 --- a/docs/en/functions/other_functions.rst +++ /dev/null @@ -1,281 +0,0 @@ -Other functions ---------------- - -hostName() -~~~~~~~~~~ -Returns a string with the name of the host that this function was performed on. For distributed processing, this is the name of the remote server host, if the function is performed on a remote server. - -visibleWidth(x) -~~~~~~~~~~~~~~~ -Calculates the approximate width when outputting values to the console in text format (tab-separated). This function is used by the system for implementing Pretty formats. - -toTypeName(x) -~~~~~~~~~~~~~ -Gets the type name. Returns a string containing the type name of the passed argument. - -blockSize() -~~~~~~~~~~~ -Gets the size of the block. -In ClickHouse, queries are always run on blocks (sets of column parts). This function allows getting the size of the block that you called it for. - -materialize(x) -~~~~~~~~~~~~~~ -Turns a constant into a full column containing just one value. -In ClickHouse, full columns and constants are represented differently in memory. Functions work differently for constant arguments and normal arguments (different code is executed), although the result is almost always the same. This function is for debugging this behavior. - -ignore(...) -~~~~~~~~~~~ -A function that accepts any arguments and always returns 0. -However, the argument is still calculated. This can be used for benchmarks. - -sleep(seconds) -~~~~~~~~~~~~~~ -Sleeps 'seconds' seconds on each data block. You can specify an integer or a floating-point number. - -currentDatabase() -~~~~~~~~~~~~~~~~~ -Returns the name of the current database. -You can use this function in table engine parameters in a CREATE TABLE query where you need to specify the database.. - -isFinite(x) -~~~~~~~~~~~ -Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is not infinite and not a NaN, otherwise 0. - -isInfinite(x) -~~~~~~~~~~~~~ -Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is infinite, otherwise 0. -Note that 0 is returned for a NaN - -isNaN(x) -~~~~~~~~ -Accepts Float32 and Float64 and returns UInt8 equal to 1 if the argument is a NaN, otherwise 0. - -hasColumnInTable(['hostname'[, 'username'[, 'password']],] 'database', 'table', 'column') -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Accepts constant String columns - database name, table name and column name. Returns constant UInt8 value, equal to 1 if column exists, -otherwise 0. If hostname is specified, the check will proceed on remote host. -If table doesn't exist than exception is thrown. -For elements of nested data structure function checks existence of column. For nested data structure 0 is returned. - -bar -~~~ -Allows building a unicode-art diagram. - -``bar(x, min, max, width)`` - Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max. -``min, max`` - Integer constants. The value must fit in Int64. -``width`` - Constant, positive number, may be a fraction. - -The band is drawn with accuracy to one eighth of a symbol. Example: - -.. code-block:: sql - - SELECT - toHour(EventTime) AS h, - count() AS c, - bar(c, 0, 600000, 20) AS bar - FROM test.hits - GROUP BY h - ORDER BY h ASC - -.. code-block:: text - - ┌──h─┬──────c─┬─bar────────────────┐ - │ 0 │ 292907 │ █████████▋ │ - │ 1 │ 180563 │ ██████ │ - │ 2 │ 114861 │ ███▋ │ - │ 3 │ 85069 │ ██▋ │ - │ 4 │ 68543 │ ██▎ │ - │ 5 │ 78116 │ ██▌ │ - │ 6 │ 113474 │ ███▋ │ - │ 7 │ 170678 │ █████▋ │ - │ 8 │ 278380 │ █████████▎ │ - │ 9 │ 391053 │ █████████████ │ - │ 10 │ 457681 │ ███████████████▎ │ - │ 11 │ 493667 │ ████████████████▍ │ - │ 12 │ 509641 │ ████████████████▊ │ - │ 13 │ 522947 │ █████████████████▍ │ - │ 14 │ 539954 │ █████████████████▊ │ - │ 15 │ 528460 │ █████████████████▌ │ - │ 16 │ 539201 │ █████████████████▊ │ - │ 17 │ 523539 │ █████████████████▍ │ - │ 18 │ 506467 │ ████████████████▊ │ - │ 19 │ 520915 │ █████████████████▎ │ - │ 20 │ 521665 │ █████████████████▍ │ - │ 21 │ 542078 │ ██████████████████ │ - │ 22 │ 493642 │ ████████████████▍ │ - │ 23 │ 400397 │ █████████████▎ │ - └────┴────────┴────────────────────┘ - -transform -~~~~~~~~~ -Transforms a value according to the explicitly defined mapping of some elements to other ones. -There are two variations of this function: - -1. ``transform(x, array_from, array_to, default)`` - -``x`` - What to transform - -``array_from`` - Constant array of values for converting. - -``array_to`` - Constant array of values to convert the values in 'from' to. - -``default`` - Constant. Which value to use if 'x' is not equal to one of the values in 'from' - -``'array_from'`` and ``'array_to'`` are arrays of the same size. - -Types: - -``transform(T, Array(T), Array(U), U) -> U`` - -``'T'`` and ``'U'`` can be numeric, string, or Date or DateTime types. -Where the same letter is indicated (T or U), for numeric types these might not be matching types, but types that have a common type. -For example, the first argument can have the Int64 type, while the second has the Array(Uint16) type. - -If the 'x' value is equal to one of the elements in the 'array_from' array, it returns the existing element (that is numbered the same) from the 'array_to' array. Otherwise, it returns 'default'. If there are multiple matching elements in 'array_from', it returns one of the matches. - -Example: - -.. code-block:: sql - - SELECT - transform(SearchEngineID, [2, 3], ['Яндекс', 'Google'], 'Остальные') AS title, - count() AS c - FROM test.hits - WHERE SearchEngineID != 0 - GROUP BY title - ORDER BY c DESC - -.. code-block:: text - - ┌─title─────┬──────c─┐ - │ Яндекс │ 498635 │ - │ Google │ 229872 │ - │ Остальные │ 104472 │ - └───────────┴────────┘ - - -2. ``transform(x, array_from, array_to)`` - -Differs from the first variation in that the 'default' argument is omitted. -If the 'x' value is equal to one of the elements in the 'array_from' array, it returns the matching element (that is numbered the same) from the 'array_to' array. Otherwise, it returns 'x'. - -Types: - -``transform(T, Array(T), Array(T)) -> T`` - -Example: - -.. code-block:: sql - - SELECT - transform(domain(Referer), ['yandex.ru', 'google.ru', 'vk.com'], ['www.yandex', 'ввв.яндекс.рф', 'example.com']) AS s, - count() AS c - FROM test.hits - GROUP BY domain(Referer) - ORDER BY count() DESC - LIMIT 10 - -.. code-block:: text - - ┌─s──────────────┬───────c─┐ - │ │ 2906259 │ - │ www.yandex │ 867767 │ - │ ███████.ru │ 313599 │ - │ mail.yandex.ru │ 107147 │ - │ ввв.яндекс.рф │ 105668 │ - │ ██████.ru │ 100355 │ - │ █████████.ru │ 65040 │ - │ news.yandex.ru │ 64515 │ - │ ██████.net │ 59141 │ - │ example.com │ 57316 │ - └────────────────┴─────────┘ - -formatReadableSize(x) -~~~~~~~~~~~~~~~~~~~~~ -Gets a size (number of bytes). Returns a string that contains rounded size with the suffix (KiB, MiB etc.). - -Example: - -.. code-block:: sql - - SELECT - arrayJoin([1, 1024, 1024*1024, 192851925]) AS filesize_bytes, - formatReadableSize(filesize_bytes) AS filesize - -.. code-block:: text - - ┌─filesize_bytes─┬─filesize───┐ - │ 1 │ 1.00 B │ - │ 1024 │ 1.00 KiB │ - │ 1048576 │ 1.00 MiB │ - │ 192851925 │ 183.92 MiB │ - └────────────────┴────────────┘ - -least(a, b) -~~~~~~~~~~~ -Returns the least element of a and b. - -greatest(a, b) -~~~~~~~~~~~~~~ -Returns the greatest element of a and b - -uptime() -~~~~~~~~ -Returns server's uptime in seconds. - -version() -~~~~~~~~~ -Returns server's version as a string. - -rowNumberInAllBlocks() -~~~~~~~~~~~~~~~~~~~~~~ -Returns an incremental row number within all blocks that were processed by this function. - -runningDifference(x) -~~~~~~~~~~~~~~~~~~~~ -Calculates the difference between consecutive values in the data block. -Result of the function depends on the order of the data in the blocks. - -It works only inside of the each processed block of data. Data splitting in the blocks is not explicitly controlled by the user. -If you specify ``ORDER BY`` in subquery and call runningDifference outside of it, you could get an expected result. - -Example: - -.. code-block:: sql - - SELECT - EventID, - EventTime, - runningDifference(EventTime) AS delta - FROM - ( - SELECT - EventID, - EventTime - FROM events - WHERE EventDate = '2016-11-24' - ORDER BY EventTime ASC - LIMIT 5 - ) - -.. code-block:: text - - ┌─EventID─┬───────────EventTime─┬─delta─┐ - │ 1106 │ 2016-11-24 00:00:04 │ 0 │ - │ 1107 │ 2016-11-24 00:00:05 │ 1 │ - │ 1108 │ 2016-11-24 00:00:05 │ 0 │ - │ 1109 │ 2016-11-24 00:00:09 │ 4 │ - │ 1110 │ 2016-11-24 00:00:10 │ 1 │ - └─────────┴─────────────────────┴───────┘ - -MACNumToString(num) -~~~~~~~~~~~~~~~~~~~ -Takes a UInt64 number. Interprets it as an MAC address in big endian. Returns a string containing the corresponding MAC address in the format AA:BB:CC:DD:EE:FF (colon-separated numbers in hexadecimal form). - -MACStringToNum(s) -~~~~~~~~~~~~~~~~~ -The reverse function of MACNumToString. If the MAC address has an invalid format, it returns 0. - -MACStringToOUI(s) -~~~~~~~~~~~~~~~~~ -Takes MAC address in the format AA:BB:CC:DD:EE:FF (colon-separated numbers in hexadecimal form). Returns first three octets as UInt64 number. If the MAC address has an invalid format, it returns 0. diff --git a/docs/en/functions/random_functions.rst b/docs/en/functions/random_functions.md similarity index 84% rename from docs/en/functions/random_functions.rst rename to docs/en/functions/random_functions.md index d9c56dbac33..915155f4af2 100644 --- a/docs/en/functions/random_functions.rst +++ b/docs/en/functions/random_functions.md @@ -1,17 +1,18 @@ -Functions for generating pseudo-random numbers ----------------------------------------------- +# Functions for generating pseudo-random numbers + Non-cryptographic generators of pseudo-random numbers are used. All the functions accept zero arguments or one argument. If an argument is passed, it can be any type, and its value is not used for anything. The only purpose of this argument is to prevent common subexpression elimination, so that two different instances of the same function return different columns with different random numbers. -rand -~~~~ +## rand + Returns a pseudo-random UInt32 number, evenly distributed among all UInt32-type numbers. Uses a linear congruential generator. -rand64 -~~~~~~ +## rand64 + Returns a pseudo-random UInt64 number, evenly distributed among all UInt64-type numbers. Uses a linear congruential generator. + diff --git a/docs/en/functions/rounding_functions.rst b/docs/en/functions/rounding_functions.md similarity index 51% rename from docs/en/functions/rounding_functions.rst rename to docs/en/functions/rounding_functions.md index ffa8295fd47..f2f8c3648cc 100644 --- a/docs/en/functions/rounding_functions.rst +++ b/docs/en/functions/rounding_functions.md @@ -1,38 +1,36 @@ -Rounding functions ------------------- +# Rounding functions -floor(x[, N]) -~~~~~~~~~~~~~ -Returns a rounder number that is less than or equal to 'x'. -A round number is a multiple of 1 / 10N, or the nearest number of the appropriate data type ``if 1 / 10N`` isn't exact. +## floor(x\[, N\]) + +Returns the largest round number that is less than or equal to x. A round number is a multiple of 1/10N, or the nearest number of the appropriate data type if 1 / 10N isn't exact. 'N' is an integer constant, optional parameter. By default it is zero, which means to round to an integer. 'N' may be negative. -Examples: ``floor(123.45, 1) = 123.4, floor(123.45, -1) = 120``. +Examples: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.` -'x' is any numeric type. The result is a number of the same type. +`x` is any numeric type. The result is a number of the same type. For integer arguments, it makes sense to round with a negative 'N' value (for non-negative 'N', the function doesn't do anything). -If rounding causes overflow (for example, ``floor(-128, -1))``, an implementation-specific result is returned. +If rounding causes overflow (for example, floor(-128, -1)), an implementation-specific result is returned. -ceil(x[, N]) -~~~~~~~~~~~~ -Returns the smallest round number that is greater than or equal to 'x'. In every other way, it is the same as the 'floor' function (see above).. +## ceil(x\[, N\]) -round(x[, N]) -~~~~~~~~~~~~~ -Returns the round number nearest to 'num', which may be less than, greater than, or equal to 'x'. -If 'x' is exactly in the middle between the nearest round numbers, one of them is returned (implementation-specific). +Returns the smallest round number that is greater than or equal to 'x'. In every other way, it is the same as the 'floor' function (see above). + +## round(x\[, N\]) + +Returns the round number nearest to 'num', which may be less than, greater than, or equal to 'x'.If 'x' is exactly in the middle between the nearest round numbers, one of them is returned (implementation-specific). The number '-0.' may or may not be considered round (implementation-specific). In every other way, this function is the same as 'floor' and 'ceil' described above. -roundToExp2(num) -~~~~~~~~~~~~~~~~ +## roundToExp2(num) + Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to the nearest (whole non-negative) degree of two. -roundDuration(num) -~~~~~~~~~~~~~~~~~~ -Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to numbers from the set: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. This function is specific to Yandex.Metrica and used for implementing the report on session length. +## roundDuration(num) + +Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to numbers from the set: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. This function is specific to Yandex.Metrica and used for implementing the report on session length + +## roundAge(num) + +Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rounds the number down to a number from the set: 18, 25, 35, 45, 55. This function is specific to Yandex.Metrica and used for implementing the report on user age. -roundAge(num) -~~~~~~~~~~~~~ -Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rounds the number down to numbers from the set: 18, 25, 35, 45, 55. This function is specific to Yandex.Metrica and used for implementing the report on user age. diff --git a/docs/en/functions/splitting_merging_functions.md b/docs/en/functions/splitting_merging_functions.md new file mode 100644 index 00000000000..3b71948c4cb --- /dev/null +++ b/docs/en/functions/splitting_merging_functions.md @@ -0,0 +1,20 @@ +# Functions for splitting and merging strings and arrays + +## splitByChar(separator, s) + +Splits a string into substrings separated by 'separator'.'separator' must be a string constant consisting of exactly one character. +Returns an array of selected substrings. Empty substrings may be selected if the separator occurs at the beginning or end of the string, or if there are multiple consecutive separators. + +## splitByString(separator, s) + +The same as above, but it uses a string of multiple characters as the separator. The string must be non-empty. + +## arrayStringConcat(arr\[, separator\]) + +Concatenates the strings listed in the array with the separator.'separator' is an optional parameter: a constant string, set to an empty string by default. +Returns the string. + +## alphaTokens(s) + +Selects substrings of consecutive bytes from the ranges a-z and A-Z.Returns an array of substrings. + diff --git a/docs/en/functions/splitting_merging_functions.rst b/docs/en/functions/splitting_merging_functions.rst deleted file mode 100644 index 3312315ef6d..00000000000 --- a/docs/en/functions/splitting_merging_functions.rst +++ /dev/null @@ -1,23 +0,0 @@ -Functions for splitting and merging strings and arrays ------------------------------------------------------- - -splitByChar(separator, s) -~~~~~~~~~~~~~~~~~~~~~~~~~ -Splits a string into substrings, using 'separator' as the separator. -'separator' must be a string constant consisting of exactly one character. -Returns an array of selected substrings. Empty substrings may be selected if the separator occurs at the beginning or end of the string, or if there are multiple consecutive separators. - -splitByString(separator, s) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The same as above, but it uses a string of multiple characters as the separator. The string must be non-empty. - -arrayStringConcat(arr[, separator]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Concatenates strings from the array elements, using 'separator' as the separator. -'separator' is a string constant, an optional parameter. By default it is an empty string. -Returns a string. - -alphaTokens(s) -~~~~~~~~~~~~~~ -Selects substrings of consecutive bytes from the range a-z and A-Z. -Returns an array of selected substrings. diff --git a/docs/en/functions/string_functions.rst b/docs/en/functions/string_functions.md similarity index 53% rename from docs/en/functions/string_functions.rst rename to docs/en/functions/string_functions.md index c54fe64a134..61e33040aa5 100644 --- a/docs/en/functions/string_functions.rst +++ b/docs/en/functions/string_functions.md @@ -1,74 +1,76 @@ -Functions for working with strings ----------------------------------- +# Functions for working with strings + +## empty -empty -~~~~~ Returns 1 for an empty string or 0 for a non-empty string. The result type is UInt8. A string is considered non-empty if it contains at least one byte, even if this is a space or a null byte. The function also works for arrays. -notEmpty -~~~~~~~~ +## notEmpty + Returns 0 for an empty string or 1 for a non-empty string. The result type is UInt8. The function also works for arrays. -length -~~~~~~ +## length + Returns the length of a string in bytes (not in characters, and not in code points). The result type is UInt64. The function also works for arrays. -lengthUTF8 -~~~~~~~~~~ +## lengthUTF8 + Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn't throw an exception). The result type is UInt64. -lower -~~~~~ +## lower + Converts ASCII Latin symbols in a string to lowercase. -upper -~~~~~ +## upper + Converts ASCII Latin symbols in a string to uppercase. -lowerUTF8 -~~~~~~~~~ -Converts a string to lowercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. It doesn't detect the language. So for Turkish the result might not be exactly correct. -If length of UTF-8 sequence is different for upper and lower case of code point, then result for that code point could be incorrect. -If value contains invalid UTF-8, the behavior is unspecified. +## lowerUTF8 -upperUTF8 -~~~~~~~~~ -Converts a string to uppercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. It doesn't detect the language. So for Turkish the result might not be exactly correct. -If length of UTF-8 sequence is different for upper and lower case of code point, then result for that code point could be incorrect. -If value contains invalid UTF-8, the behavior is unspecified. +Converts a string to lowercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. +It doesn't detect the language. So for Turkish the result might not be exactly correct. +If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. +If the string contains a set of bytes that is not UTF-8, then the behavior is undefined. + +## upperUTF8 + +Converts a string to uppercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. +It doesn't detect the language. So for Turkish the result might not be exactly correct. +If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. +If the string contains a set of bytes that is not UTF-8, then the behavior is undefined. + +## reverse -reverse -~~~~~~~ Reverses the string (as a sequence of bytes). -reverseUTF8 -~~~~~~~~~~~ +## reverseUTF8 + Reverses a sequence of Unicode code points, assuming that the string contains a set of bytes representing a UTF-8 text. Otherwise, it does something else (it doesn't throw an exception). -concat(s1, s2, ...) -~~~~~~~~~~~~~~~~~~~ -Concatenates strings from the function arguments, without a separator. +## concat(s1, s2, ...) + +Concatenates the strings listed in the arguments, without a separator. + +## substring(s, offset, length) -substring(s, offset, length) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Returns a substring starting with the byte from the 'offset' index that is 'length' bytes long. Character indexing starts from one (as in standard SQL). The 'offset' and 'length' arguments must be constants. -substringUTF8(s, offset, length) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## substringUTF8(s, offset, length) + The same as 'substring', but for Unicode code points. Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn't throw an exception). -appendTrailingCharIfAbsent(s, c) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If the ``s`` string is non-empty and does not contain the ``c`` character at the end, it appends the ``c`` character to the end. +## appendTrailingCharIfAbsent(s, c) + +If the 's' string is non-empty and does not contain the 'c' character at the end, it appends the 'c' character to the end. + +## convertCharset(s, from, to) + +Returns the string 's' that was converted from the encoding in 'from' to the encoding in 'to'. -convertCharset(s, from, to) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns a string with the data s (encoded as from charset) that was converted to the to charset. diff --git a/docs/en/functions/string_replace_functions.md b/docs/en/functions/string_replace_functions.md new file mode 100644 index 00000000000..d3773504278 --- /dev/null +++ b/docs/en/functions/string_replace_functions.md @@ -0,0 +1,79 @@ +# Functions for searching and replacing in strings + +## replaceOne(haystack, pattern, replacement) + +Replaces the first occurrence, if it exists, of the 'pattern' substring in 'haystack' with the 'replacement' substring. +Hereafter, 'pattern' and 'replacement' must be constants. + +## replaceAll(haystack, pattern, replacement) + +Replaces all occurrences of the 'pattern' substring in 'haystack' with the 'replacement' substring. + +## replaceRegexpOne(haystack, pattern, replacement) + +Replacement using the 'pattern' regular expression. A re2 regular expression. +Replaces only the first occurrence, if it exists. +A pattern can be specified as 'replacement'. This pattern can include substitutions `\0-\9`. +The substitution `\0` includes the entire regular expression. Substitutions `\1-\9` correspond to the subpattern numbers.To use the `\` character in a template, escape it using `\`. +Also keep in mind that a string literal requires an extra escape. + +Example 1. Converting the date to American format: + +```sql +SELECT DISTINCT + EventDate, + replaceRegexpOne(toString(EventDate), '(\\d{4})-(\\d{2})-(\\d{2})', '\\2/\\3/\\1') AS res +FROM test.hits +LIMIT 7 +FORMAT TabSeparated +``` + +```text +2014-03-17 03/17/2014 +2014-03-18 03/18/2014 +2014-03-19 03/19/2014 +2014-03-20 03/20/2014 +2014-03-21 03/21/2014 +2014-03-22 03/22/2014 +2014-03-23 03/23/2014 +``` + +Example 2. Copying a string ten times: + +```sql +SELECT replaceRegexpOne('Hello, World!', '.*', '\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0') AS res +``` + +```text +┌─res────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World! │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## replaceRegexpAll(haystack, pattern, replacement) + +This does the same thing, but replaces all the occurrences. Example: + +```sql +SELECT replaceRegexpAll('Hello, World!', '.', '\\0\\0') AS res +``` + +```text +┌─res────────────────────────┐ +│ HHeelllloo,, WWoorrlldd!! │ +└────────────────────────────┘ +``` + +As an exception, if a regular expression worked on an empty substring, the replacement is not made more than once. +Example: + +```sql +SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res +``` + +```text +┌─res─────────────────┐ +│ here: Hello, World! │ +└─────────────────────┘ +``` + diff --git a/docs/en/functions/string_replace_functions.rst b/docs/en/functions/string_replace_functions.rst deleted file mode 100644 index c7f719c2b1f..00000000000 --- a/docs/en/functions/string_replace_functions.rst +++ /dev/null @@ -1,80 +0,0 @@ -Functions for searching and replacing in strings ------------------------------------------------- - -replaceOne(haystack, pattern, replacement) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Replaces the first occurrence, if it exists, of the 'pattern' substring in 'haystack' with the 'replacement' substring. -Hereafter, 'pattern' and 'replacement' must be constants. - -replaceAll(haystack, pattern, replacement) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Replaces all occurrences of the 'pattern' substring in 'haystack' with the 'replacement' substring. - -replaceRegexpOne(haystack, pattern, replacement) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Replacement using the 'pattern' regular expression. A re2 regular expression. Replaces only the first occurrence, if it exists. -A pattern can be specified as 'replacement'. This pattern can include substitutions \0-\9\. -The substitution \0 includes the entire regular expression. -The substitutions \1-\9 include the subpattern corresponding to the number. -In order to specify the \ symbol in a pattern, you must use a \ symbol to escape it. -Also keep in mind that a string literal requires an extra escape. - -Example 1. Converting the date to American format: - -.. code-block:: sql - - SELECT DISTINCT - EventDate, - replaceRegexpOne(toString(EventDate), '(\\d{4})-(\\d{2})-(\\d{2})', '\\2/\\3/\\1') AS res - FROM test.hits - LIMIT 7 - FORMAT TabSeparated - -.. code-block:: text - - 2014-03-17 03/17/2014 - 2014-03-18 03/18/2014 - 2014-03-19 03/19/2014 - 2014-03-20 03/20/2014 - 2014-03-21 03/21/2014 - 2014-03-22 03/22/2014 - 2014-03-23 03/23/2014 - -Example 2. Copy the string ten times: - -.. code-block:: sql - - SELECT replaceRegexpOne('Hello, World!', '.*', '\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0') AS res - -.. code-block:: text - - ┌─res────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World! │ - └────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ - -replaceRegexpAll(haystack, pattern, replacement) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This does the same thing, but replaces all the occurrences. Example: - -.. code-block:: sql - - SELECT replaceRegexpAll('Hello, World!', '.', '\\0\\0') AS res - -.. code-block:: text - - ┌─res────────────────────────┐ - │ HHeelllloo,, WWoorrlldd!! │ - └────────────────────────────┘ - -As an exception, if a regular expression worked on an empty substring, the replacement is not made more than once. -Example: - -.. code-block:: sql - - SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res - -.. code-block:: text - - ┌─res─────────────────┐ - │ here: Hello, World! │ - └─────────────────────┘ diff --git a/docs/en/functions/string_search_functions.md b/docs/en/functions/string_search_functions.md new file mode 100644 index 00000000000..ba3e53d4ee8 --- /dev/null +++ b/docs/en/functions/string_search_functions.md @@ -0,0 +1,52 @@ +# Functions for searching strings + +The search is case-sensitive in all these functions. +The search substring or regular expression must be a constant in all these functions. + +## position(haystack, needle) + +Search for the 'needle' substring in the 'haystack' string. +Returns the position (in bytes) of the found substring, starting from 1, or returns 0 if the substring was not found. +It has also chimpanzees. + +## positionUTF8(haystack, needle) + +The same as 'position', but the position is returned in Unicode code points. Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn't throw an exception). +There is also a positionCaseInsensitiveUTF8 function. + +## match(haystack, pattern) + +Checks whether the string matches the 'pattern' regular expression. A re2 regular expression. +Returns 0 if it doesn't match, or 1 if it matches. + +Note that the backslash symbol (`\`) is used for escaping in the regular expression. The same symbol is used for escaping in string literals. So in order to escape the symbol in a regular expression, you must write two backslashes (\\) in a string literal. + +The regular expression works with the string as if it is a set of bytes. The regular expression can't contain null bytes. +For patterns to search for substrings in a string, it is better to use LIKE or 'position', since they work much faster. + +## extract(haystack, pattern) + +Extracts a fragment of a string using a regular expression. If 'haystack' doesn't match the 'pattern' regex, an empty string is returned. If the regex doesn't contain subpatterns, it takes the fragment that matches the entire regex. Otherwise, it takes the fragment that matches the first subpattern. + +## extractAll(haystack, pattern) + +Extracts all the fragments of a string using a regular expression. If 'haystack' doesn't match the 'pattern' regex, an empty string is returned. Returns an array of strings consisting of all matches to the regex. In general, the behavior is the same as the 'extract' function (it takes the first subpattern, or the entire expression if there isn't a subpattern). + +## like(haystack, pattern), haystack LIKE pattern operator + +Checks whether a string matches a simple regular expression. +The regular expression can contain the metasymbols `%` and `_`. + +``% indicates any quantity of any bytes (including zero characters). + +`_` indicates any one byte. + +Use the backslash (`\`) for escaping metasymbols. See the note on escaping in the description of the 'match' function. + +For regular expressions like `%needle%`, the code is more optimal and works as fast as the `position` function. +For other regular expressions, the code is the same as for the 'match' function. + +## notLike(haystack, pattern), haystack NOT LIKE pattern operator + +The same thing as 'like', but negative. + diff --git a/docs/en/functions/string_search_functions.rst b/docs/en/functions/string_search_functions.rst deleted file mode 100644 index 942efbdc29f..00000000000 --- a/docs/en/functions/string_search_functions.rst +++ /dev/null @@ -1,52 +0,0 @@ -Functions for searching strings -------------------------------- -The search is case-sensitive in all these functions. -The search substring or regular expression must be a constant in all these functions. - -position(haystack, needle) -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Searches for the 'needle' substring in the 'haystack' string. -Returns the position (in bytes) of the found substring, starting from 1, or returns 0 if the substring was not found. -There's also positionCaseInsensitive function. - -positionUTF8(haystack, needle) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The same as 'position', but the position is returned in Unicode code points. Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn't throw an exception). -There's also positionCaseInsensitiveUTF8 function. - -match(haystack, pattern) -~~~~~~~~~~~~~~~~~~~~~~~~ -Checks whether the string matches the 'pattern' regular expression. -The regular expression is re2. -Returns 0 if it doesn't match, or 1 if it matches. - -Note that the backslash symbol (``\``) is used for escaping in the regular expression. The same symbol is used for escaping in string literals. -So in order to escape the symbol in a regular expression, you must write two backslashes (``\\``) in a string literal. - -The regular expression works with the string as if it is a set of bytes. -The regular expression can't contain null bytes. -For patterns to search for substrings in a string, it is better to use LIKE or 'position', since they work much faster. - -extract(haystack, pattern) -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Extracts a fragment of a string using a regular expression. If 'haystack' doesn't match the 'pattern' regex, an empty string is returned. If the regex doesn't contain subpatterns, it takes the fragment that matches the entire regex. Otherwise, it takes the fragment that matches the first subpattern. - -extractAll(haystack, pattern) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Extracts all the fragments of a string using a regular expression. If 'haystack' doesn't match the 'pattern' regex, an empty string is returned. Returns an array of strings consisting of all matches to the regex. In general, the behavior is the same as the 'extract' function (it takes the first subpattern, or the entire expression if there isn't a subpattern). - -like(haystack, pattern), оператор haystack LIKE pattern -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Checks whether a string matches a simple regular expression. The regular expression can contain the metasymbols ``%`` and ``_``. - -``%`` indicates any quantity of any bytes (including zero characters). - -``_`` indicates any one byte. - -Use the backslash (``\``) for escaping metasymbols. See the note on escaping in the description of the 'match' function. - -For regular expressions like%needle%, the code is more optimal and works as fast as the 'position' function. For other regular expressions, the code is the same as for the 'match' function. - -notLike(haystack, pattern), оператор haystack NOT LIKE pattern -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The same thing as 'like', but negative. diff --git a/docs/en/functions/type_conversion_functions.md b/docs/en/functions/type_conversion_functions.md new file mode 100644 index 00000000000..903f370e2b2 --- /dev/null +++ b/docs/en/functions/type_conversion_functions.md @@ -0,0 +1,121 @@ + + +# Type conversion functions + +## toUInt8, toUInt16, toUInt32, toUInt64 + +## toInt8, toInt16, toInt32, toInt64 + +## toFloat32, toFloat64 + +## toUInt8OrZero, toUInt16OrZero, toUInt32OrZero, toUInt64OrZero, toInt8OrZero, toInt16OrZero, toInt32OrZero, toInt64OrZero, toFloat32OrZero, toFloat64OrZero + +## toDate, toDateTime + +## toString + +Functions for converting between numbers, strings (but not fixed strings), dates, and dates with times. +All these functions accept one argument. + +When converting to or from a string, the value is formatted or parsed using the same rules as for the TabSeparated format (and almost all other text formats). If the string can't be parsed, an exception is thrown and the request is canceled. + +When converting dates to numbers or vice versa, the date corresponds to the number of days since the beginning of the Unix epoch. +When converting dates with times to numbers or vice versa, the date with time corresponds to the number of seconds since the beginning of the Unix epoch. + +The date and date-with-time formats for the toDate/toDateTime functions are defined as follows: + +```text +YYYY-MM-DD +YYYY-MM-DD hh:mm:ss +``` + +As an exception, if converting from UInt32, Int32, UInt64, or Int64 numeric types to Date, and if the number is greater than or equal to 65536, the number is interpreted as a Unix timestamp (and not as the number of days) and is rounded to the date. This allows support for the common occurrence of writing 'toDate(unix_timestamp)', which otherwise would be an error and would require writing the more cumbersome 'toDate(toDateTime(unix_timestamp))'. + +Conversion between a date and date with time is performed the natural way: by adding a null time or dropping the time. + +Conversion between numeric types uses the same rules as assignments between different numeric types in C++. + +Additionally, the toString function of the DateTime argument can take a second String argument containing the name of the time zone. Example: `Asia/Yekaterinburg` In this case, the time is formatted according to the specified time zone. + +```sql +SELECT + now() AS now_local, + toString(now(), 'Asia/Yekaterinburg') AS now_yekat +``` + +```text +┌───────────now_local─┬─now_yekat───────────┐ +│ 2016-06-15 00:11:21 │ 2016-06-15 02:11:21 │ +└─────────────────────┴─────────────────────┘ +``` + +Also see the `toUnixTimestamp` function. + +## toFixedString(s, N) + +Converts a String type argument to a FixedString(N) type (a string with fixed length N). N must be a constant. +If the string has fewer bytes than N, it is passed with null bytes to the right. If the string has more bytes than N, an exception is thrown. + +## toStringCutToZero(s) + +Accepts a String or FixedString argument. Returns the String with the content truncated at the first zero byte found. + +Example: + +```sql +SELECT toFixedString('foo', 8) AS s, toStringCutToZero(s) AS s_cut +``` + +```text +┌─s─────────────┬─s_cut─┐ +│ foo\0\0\0\0\0 │ foo │ +└───────────────┴───────┘ +``` + +```sql +SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut +``` + +```text +┌─s──────────┬─s_cut─┐ +│ foo\0bar\0 │ foo │ +└────────────┴───────┘ +``` + +## reinterpretAsUInt8, reinterpretAsUInt16, reinterpretAsUInt32, reinterpretAsUInt64 + +## reinterpretAsInt8, reinterpretAsInt16, reinterpretAsInt32, reinterpretAsInt64 + +## reinterpretAsFloat32, reinterpretAsFloat64 + +## reinterpretAsDate, reinterpretAsDateTime + +These functions accept a string and interpret the bytes placed at the beginning of the string as a number in host order (little endian). If the string isn't long enough, the functions work as if the string is padded with the necessary number of null bytes. If the string is longer than needed, the extra bytes are ignored. A date is interpreted as the number of days since the beginning of the Unix Epoch, and a date with time is interpreted as the number of seconds since the beginning of the Unix Epoch. + +## reinterpretAsString + +This function accepts a number or date or date with time, and returns a string containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a string that is one byte long. + +## CAST(x, t) + +Converts 'x' to the 't' data type. The syntax CAST(x AS t) is also supported. + +Example: + +```sql +SELECT + '2016-06-15 23:00:00' AS timestamp, + CAST(timestamp AS DateTime) AS datetime, + CAST(timestamp AS Date) AS date, + CAST(timestamp, 'String') AS string, + CAST(timestamp, 'FixedString(22)') AS fixed_string +``` + +```text +┌─timestamp───────────┬────────────datetime─┬───────date─┬─string──────────────┬─fixed_string──────────────┐ +│ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00\0\0\0 │ +└─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ +``` + +Conversion to FixedString (N) only works for arguments of type String or FixedString (N). + diff --git a/docs/en/functions/type_conversion_functions.rst b/docs/en/functions/type_conversion_functions.rst deleted file mode 100644 index 260ce292a2c..00000000000 --- a/docs/en/functions/type_conversion_functions.rst +++ /dev/null @@ -1,132 +0,0 @@ -Type conversion functions -------------------------- - -toUInt8, toUInt16, toUInt32, toUInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -toInt8, toInt16, toInt32, toInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -toFloat32, toFloat64 -~~~~~~~~~~~~~~~~~~~~ - -toUInt8OrZero, toUInt16OrZero, toUInt32OrZero, toUInt64OrZero, toInt8OrZero, toInt16OrZero, toInt32OrZero, toInt64OrZero, toFloat32OrZero, toFloat64OrZero -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -toDate, toDateTime -~~~~~~~~~~~~~~~~~~ - -toString -~~~~~~~~ -Functions for converting between numbers, strings (but not fixed strings), dates, and dates with times. All these functions accept one argument. - -When converting to or from a string, the value is formatted or parsed using the same rules as for the TabSeparated format (and almost all other text formats). If the string can't be parsed, an exception is thrown and the request is canceled. - -When converting dates to numbers or vice versa, the date corresponds to the number of days since the beginning of the Unix epoch. -When converting dates with times to numbers or vice versa, the date with time corresponds to the number of seconds since the beginning of the Unix epoch. - -Formats of date and date with time for toDate/toDateTime functions are defined as follows: - -.. code-block:: text - - YYYY-MM-DD - YYYY-MM-DD hh:mm:ss - -As an exception, if converting from UInt32, Int32, UInt64, or Int64 type numbers to Date, and if the number is greater than or equal to 65536, the number is interpreted as a Unix timestamp (and not as the number of days) and is rounded to the date. This allows support for the common occurrence of writing 'toDate(unix_timestamp)', which otherwise would be an error and would require writing the more cumbersome 'toDate(toDateTime(unix_timestamp))'. - -Conversion between a date and date with time is performed the natural way: by adding a null time or dropping the time. - -Conversion between numeric types uses the same rules as assignments between different numeric types in C++. - -To do transformations on DateTime in given time zone, pass second argument with time zone name: - -.. code-block:: sql - - SELECT - now() AS now_local, - toString(now(), 'Asia/Yekaterinburg') AS now_yekat - -.. code-block:: text - - ┌───────────now_local─┬─now_yekat───────────┐ - │ 2016-06-15 00:11:21 │ 2016-06-15 02:11:21 │ - └─────────────────────┴─────────────────────┘ - -To format DateTime in given time zone: - -.. code-block:: text - - toString(now(), 'Asia/Yekaterinburg') - -To get unix timestamp for string with datetime in specified time zone: - -.. code-block:: text - - toUnixTimestamp('2000-01-01 00:00:00', 'Asia/Yekaterinburg') - -toFixedString(s, N) -~~~~~~~~~~~~~~~~~~~ -Converts a String type argument to a FixedString(N) type (a string with fixed length N). N must be a constant. If the string has fewer bytes than N, it is passed with null bytes to the right. If the string has more bytes than N, an exception is thrown. - -toStringCutToZero(s) -~~~~~~~~~~~~~~~~~~~~ -Accepts a String or FixedString argument. Returns a String that is cut to a first null byte occurrence. - -Example: - -.. code-block:: sql - - SELECT toFixedString('foo', 8) AS s, toStringCutToZero(s) AS s_cut - -.. code-block:: text - - ┌─s─────────────┬─s_cut─┐ - │ foo\0\0\0\0\0 │ foo │ - └───────────────┴───────┘ - - SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut - - ┌─s──────────┬─s_cut─┐ - │ foo\0bar\0 │ foo │ - └────────────┴───────┘ - -reinterpretAsUInt8, reinterpretAsUInt16, reinterpretAsUInt32, reinterpretAsUInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -reinterpretAsInt8, reinterpretAsInt16, reinterpretAsInt32, reinterpretAsInt64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -reinterpretAsFloat32, reinterpretAsFloat64 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -reinterpretAsDate, reinterpretAsDateTime -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -These functions accept a string and interpret the bytes placed at the beginning of the string as a number in host order (little endian). If the string isn't long enough, the functions work as if the string is padded with the necessary number of null bytes. If the string is longer than needed, the extra bytes are ignored. A date is interpreted as the number of days since the beginning of the Unix Epoch, and a date with time is interpreted as the number of seconds since the beginning of the Unix Epoch. - -reinterpretAsString -~~~~~~~~~~~~~~~~~~~ -This function accepts a number or date or date with time, and returns a string containing bytes representing the corresponding value in host order (little endian). Null bytes are dropped from the end. For example, a UInt32 type value of 255 is a string that is one byte long. - -CAST(x, t) -~~~~~~~~~~ -Casts x to the t data type. -The syntax ``CAST(x AS t)`` is also supported. - -Example: - -.. code-block:: sql - - SELECT - '2016-06-15 23:00:00' AS timestamp, - CAST(timestamp AS DateTime) AS datetime, - CAST(timestamp AS Date) AS date, - CAST(timestamp, 'String') AS string, - CAST(timestamp, 'FixedString(22)') AS fixed_string - -.. code-block:: text - - ┌─timestamp───────────┬────────────datetime─┬───────date─┬─string──────────────┬─fixed_string──────────────┐ - │ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00 │ 2016-06-15 │ 2016-06-15 23:00:00 │ 2016-06-15 23:00:00\0\0\0 │ - └─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ - -Casting to FixedString(N) works only for String and FixedString(N). diff --git a/docs/en/functions/url_functions.md b/docs/en/functions/url_functions.md new file mode 100644 index 00000000000..ef0c7e08158 --- /dev/null +++ b/docs/en/functions/url_functions.md @@ -0,0 +1,121 @@ +# Functions for working with URLs + +All these functions don't follow the RFC. They are maximally simplified for improved performance. + +## Functions that extract part of a URL + +If there isn't anything similar in a URL, an empty string is returned. + +### protocol + +Returns the protocol. Examples: http, ftp, mailto, magnet... + +### domain + +Gets the domain. + +### domainWithoutWWW + +Returns the domain and removes no more than one 'www.' from the beginning of it, if present. + +### topLevelDomain + +Returns the top-level domain. Example: .ru. + +### firstSignificantSubdomain + +Returns the "first significant subdomain". This is a non-standard concept specific to Yandex.Metrica. The first significant subdomain is a second-level domain if it is 'com', 'net', 'org', or 'co'. Otherwise, it is a third-level domain. For example, firstSignificantSubdomain ('') = 'yandex ', firstSignificantSubdomain ('') = 'yandex '. The list of "insignificant" second-level domains and other implementation details may change in the future. + +### cutToFirstSignificantSubdomain + +Returns the part of the domain that includes top-level subdomains up to the "first significant subdomain" (see the explanation above). + +For example, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. + +### path + +Returns the path. Example: `/top/news.html` The path does not include the query string. + +### pathFull + +The same as above, but including query string and fragment. Example: /top/news.html?page=2\#comments + +### queryString + +Returns the query string. Example: page=1&lr=213. query-string does not include the initial question mark, as well as \# and everything after \#. + +### fragment + +Returns the fragment identifier. fragment does not include the initial hash symbol. + +### queryStringAndFragment + +Returns the query string and fragment identifier. Example: page=1\#29390. + +### extractURLParameter(URL, name) + +Returns the value of the 'name' parameter in the URL, if present. Otherwise, an empty string. If there are many parameters with this name, it returns the first occurrence. This function works under the assumption that the parameter name is encoded in the URL exactly the same way as in the passed argument. + +### extractURLParameters(URL) + +Returns an array of name=value strings corresponding to the URL parameters. The values are not decoded in any way. + +### extractURLParameterNames(URL) + +Returns an array of name strings corresponding to the names of URL parameters. The values are not decoded in any way. + +### URLHierarchy(URL) + +Returns an array containing the URL, truncated at the end by the symbols /,? in the path and query-string. Consecutive separator characters are counted as one. The cut is made in the position after all the consecutive separator characters. Example: + +### URLPathHierarchy(URL) + +The same as above, but without the protocol and host in the result. The / element (root) is not included. Example: the function is used to implement tree reports the URL in Yandex. Metric. + +```text +URLPathHierarchy('https://example.com/browse/CONV-6788') = +[ + '/browse/', + '/browse/CONV-6788' +] +``` + +### decodeURLComponent(URL) + +Returns the decoded URL. +Example: + +```sql +SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS DecodedURL; +``` + +```text +┌─DecodedURL─────────────────────────────┐ +│ http://127.0.0.1:8123/?query=SELECT 1; │ +└────────────────────────────────────────┘ +``` + +## Functions that remove part of a URL. + +If the URL doesn't have anything similar, the URL remains unchanged. + +### cutWWW + +Removes no more than one 'www.' from the beginning of the URL's domain, if present. + +### cutQueryString + +Removes query string. The question mark is also removed. + +### cutFragment + +Removes the fragment identifier. The number sign is also removed. + +### cutQueryStringAndFragment + +Removes the query string and fragment identifier. The question mark and number sign are also removed. + +### cutURLParameter(URL, name) + +Removes the 'name' URL parameter, if present. This function works under the assumption that the parameter name is encoded in the URL exactly the same way as in the passed argument. + diff --git a/docs/en/functions/url_functions.rst b/docs/en/functions/url_functions.rst deleted file mode 100644 index 5e5d0caf160..00000000000 --- a/docs/en/functions/url_functions.rst +++ /dev/null @@ -1,123 +0,0 @@ -Functions for working with URLs -------------------------------- - -All these functions don't follow the RFC. They are maximally simplified for improved performance. - -Functions tat extract part of the URL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If there isn't anything similar in a URL, an empty string is returned. - -protocol -"""""""" -Selects the protocol. Examples: http, ftp, mailto, magnet... - -domain -"""""" -Selects the domain. - -domainWithoutWWW -"""""""""""""""" -Selects the domain and removes no more than one 'www.' from the beginning of it, if present. - -topLevelDomain -"""""""""""""" -Selects the top-level domain. Example: .ru. - -firstSignificantSubdomain -""""""""""""""""""""""""" -Selects the "first significant subdomain". This is a non-standard concept specific to Yandex.Metrica. The first significant subdomain is a second-level domain if it is 'com', 'net', 'org', or 'co'. Otherwise, it is a third-level domain. For example, firstSignificantSubdomain('https://news.yandex.ru/') = 'yandex', firstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex'. The list of "insignificant" second-level domains and other implementation details may change in the future. - -cutToFirstSignificantSubdomain -"""""""""""""""""""""""""""""" -Selects the part of the domain that includes top-level subdomains up to the "first significant subdomain" (see the explanation above). - -For example, ``cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'``. - -path -"""" -Selects the path. Example: /top/news.html The path does not include the query-string. - -pathFull -"""""""" -The same as above, but including query-string and fragment. Example: /top/news.html?page=2#comments - -queryString -""""""""""" -Selects the query-string. Example: page=1&lr=213. query-string does not include the first question mark, or # and everything that comes after #. - -fragment -"""""""" -Selects the fragment identifier. fragment does not include the first number sign (#). - -queryStringAndFragment -"""""""""""""""""""""" -Selects the query-string and fragment identifier. Example: page=1#29390. - -extractURLParameter(URL, name) -"""""""""""""""""""""""""""""" -Selects the value of the 'name' parameter in the URL, if present. Otherwise, selects an empty string. If there are many parameters with this name, it returns the first occurrence. This function works under the assumption that the parameter name is encoded in the URL in exactly the same way as in the argument passed. - -extractURLParameters(URL) -""""""""""""""""""""""""" -Gets an array of name=value strings corresponding to the URL parameters. The values are not decoded in any way. - -extractURLParameterNames(URL) -""""""""""""""""""""""""""""" -Gets an array of name=value strings corresponding to the names of URL parameters. The values are not decoded in any way. - -URLHierarchy(URL) -""""""""""""""""" -Gets an array containing the URL trimmed to the ``/``, ``?`` characters in the path and query-string. Consecutive separator characters are counted as one. The cut is made in the position after all the consecutive separator characters. Example: - -URLPathHierarchy(URL) -""""""""""""""""""""" -The same thing, but without the protocol and host in the result. The / element (root) is not included. Example: -This function is used for implementing tree-view reports by URL in Yandex.Metrica. - -.. code-block:: text - - URLPathHierarchy('https://example.com/browse/CONV-6788') = - [ - '/browse/', - '/browse/CONV-6788' - ] - -decodeURLComponent(URL) -""""""""""""""""""""""" -Returns a URL-decoded URL. - -Example: - -.. code-block:: sql - - SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS DecodedURL; - -.. code-block:: text - - ┌─DecodedURL─────────────────────────────┐ - │ http://127.0.0.1:8123/?query=SELECT 1; │ - └────────────────────────────────────────┘ - -Functions that remove part of a URL. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If the URL doesn't have anything similar, the URL remains unchanged. - -cutWWW -"""""" -Removes no more than one 'www.' from the beginning of the URL's domain, if present. - -cutQueryString -"""""""""""""" -Removes the query-string. The question mark is also removed.. - -cutFragment -""""""""""" -Removes the fragment identifier. The number sign is also removed. - -cutQueryStringAndFragment -""""""""""""""""""""""""" -Removes the query-string and fragment identifier. The question mark and number sign are also removed. - -cutURLParameter(URL, name) -"""""""""""""""""""""""""" -Removes the URL parameter named 'name', if present. This function works under the assumption that the parameter name is encoded in the URL exactly the same way as in the passed argument. diff --git a/docs/en/functions/ym_dict_functions.md b/docs/en/functions/ym_dict_functions.md new file mode 100644 index 00000000000..540b5dd601a --- /dev/null +++ b/docs/en/functions/ym_dict_functions.md @@ -0,0 +1,119 @@ +# Functions for working with Yandex.Metrica dictionaries + +In order for the functions below to work, the server config must specify the paths and addresses for getting all the Yandex.Metrica dictionaries. The dictionaries are loaded at the first call of any of these functions. If the reference lists can't be loaded, an exception is thrown. + +For information about creating reference lists, see the section "Dictionaries". + +## Multiple geobases + +ClickHouse supports working with multiple alternative geobases (regional hierarchies) simultaneously, in order to support various perspectives on which countries certain regions belong to. + +The 'clickhouse-server' config specifies the file with the regional hierarchy::`/opt/geo/regions_hierarchy.txt` + +Besides this file, it also searches for files nearby that have the _ symbol and any suffix appended to the name (before the file extension). +For example, it will also find the file `/opt/geo/regions_hierarchy_ua.txt`, if present. + +`ua` is called the dictionary key. For a dictionary without a suffix, the key is an empty string. + +All the dictionaries are re-loaded in runtime (once every certain number of seconds, as defined in the builtin_dictionaries_reload_interval config parameter, or once an hour by default). However, the list of available dictionaries is defined one time, when the server starts. + +All functions for working with regions have an optional argument at the end – the dictionary key. It is referred to as the geobase. +Example: + +```text +regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt +``` + +### regionToCity(id[, geobase]) + +Accepts a UInt32 number – the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. + +### regionToArea(id\[, geobase\]) + +Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'. + +```sql +SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 +``` + +```text +┌─regionToName(regionToArea(toUInt32(number), \'ua\'))─┐ +│ │ +│ Moscow and Moscow region │ +│ St. Petersburg and Leningrad region │ +│ Belgorod region │ +│ Ivanovsk region │ +│ Kaluga region │ +│ Kostroma region │ +│ Kursk region │ +│ Lipetsk region │ +│ Orlov region │ +│ Ryazan region │ +│ Smolensk region │ +│ Tambov region │ +│ Tver region │ +│ Tula region │ +└──────────────────────────────────────────────────────┘ +``` + +### regionToDistrict(id[, geobase]) + +Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'. + +```sql +SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 +``` + +```text +┌─regionToName(regionToDistrict(toUInt32(number), \'ua\'))─┐ +│ │ +│ Central federal district │ +│ Northwest federal district │ +│ South federal district │ +│ North Caucases federal district │ +│ Privolga federal district │ +│ Ural federal district │ +│ Siberian federal district │ +│ Far East federal district │ +│ Scotland │ +│ Faroe Islands │ +│ Flemish region │ +│ Brussels capital region │ +│ Wallonia │ +│ Federation of Bosnia and Herzegovina │ +└──────────────────────────────────────────────────────────┘ +``` + +### regionToCountry(id[, geobase]) + +Converts a region to a country. In every other way, this function is the same as 'regionToCity'. +Example: `regionToCountry(toUInt32(213)) = 225` converts Moscow (213) to Russia (225). + +### regionToContinent(id[, geobase]) + +Converts a region to a continent. In every other way, this function is the same as 'regionToCity'. +Example: `regionToContinent(toUInt32(213)) = 10001` converts Moscow (213) to Eurasia (10001). + +### regionToPopulation(id[, geobase]) + +Gets the population for a region. +The population can be recorded in files with the geobase. See the section "External dictionaries". +If the population is not recorded for the region, it returns 0. +In the Yandex geobase, the population might be recorded for child regions, but not for parent regions. + +### regionIn(lhs, rhs[, geobase]) + +Checks whether a 'lhs' region belongs to a 'rhs' region. Returns a UInt8 number equal to 1 if it belongs, or 0 if it doesn't belong. +The relationship is reflexive – any region also belongs to itself. + +### regionHierarchy(id\[, geobase\]) + +Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. +Example: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. + +### regionToName(id\[, lang\]) + +Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language 'ru' is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn't exist, an empty string is returned. + +`ua` and `uk` both mean Ukrainian. + diff --git a/docs/en/functions/ym_dict_functions.rst b/docs/en/functions/ym_dict_functions.rst deleted file mode 100644 index 69d1171aaea..00000000000 --- a/docs/en/functions/ym_dict_functions.rst +++ /dev/null @@ -1,125 +0,0 @@ -Functions for working with Yandex.Metrica dictionaries ------------------------------------------------------- -In order for the functions below to work, the server config must specify the paths and addresses for getting all the Yandex.Metrica dictionaries. The dictionaries are loaded at the first call of any of these functions. If the reference lists can't be loaded, an exception is thrown. - -For information about creating reference lists, see the section "Dictionaries". - -Multiple geobases -~~~~~~~~~~~~~~~~~ -ClickHouse supports working with multiple alternative geobases (regional hierarchies) simultaneously, in order to support various perspectives on which countries certain regions belong to. - -The 'clickhouse-server' config specifies the file with the regional hierarchy: -``/opt/geo/regions_hierarchy.txt`` - -Besides this file, it also searches for files nearby that have the _ symbol and any suffix appended to the name (before the file extension). -For example, it will also find the file ``/opt/geo/regions_hierarchy_ua.txt``, if present. - -``ua`` is called the dictionary key. For a dictionary without a suffix, the key is an empty string. - -All the dictionaries are re-loaded in runtime (once every certain number of seconds, as defined in the builtin_dictionaries_reload_interval config parameter, or once an hour by default). However, the list of available dictionaries is defined one time, when the server starts. - -All functions for working with regions have an optional argument at the end - the dictionary key. It is indicated as the geobase. -Example: - -.. code-block:: text - - regionToCountry(RegionID) - Uses the default dictionary: /opt/geo/regions_hierarchy.txt - regionToCountry(RegionID, '') - Uses the default dictionary: /opt/geo/regions_hierarchy.txt - regionToCountry(RegionID, 'ua') - Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt - -regionToCity(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Accepts a UInt32 number - the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. - -regionToArea(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'. - -.. code-block:: sql - - SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'), 'en') - FROM system.numbers - LIMIT 15 - -.. code-block:: text - - ┌─regionToName(regionToArea(toUInt32(number), \'ua\'), \'en\')─┐ - │ │ - │ Moscow and Moscow region │ - │ Saint-Petersburg and Leningradskaya oblast │ - │ Belgorod District │ - │ Ivanovo district │ - │ Kaluga District │ - │ Kostroma District │ - │ Kursk District │ - │ Lipetsk District │ - │ Orel District │ - │ Ryazhan District │ - │ Smolensk District │ - │ Tambov District │ - │ Tver District │ - │ Tula District │ - └──────────────────────────────────────────────────────────────┘ - -regionToDistrict(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'. - -.. code-block:: sql - - SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'), 'en') - FROM system.numbers - LIMIT 15 - -.. code-block:: text - - ┌─regionToName(regionToDistrict(toUInt32(number), \'ua\'), \'en\')─┐ - │ │ - │ Central │ - │ Northwest │ - │ South │ - │ North Kavkaz │ - │ Volga Region │ - │ Ural │ - │ Siberian │ - │ Far East │ - │ Scotland │ - │ Faroe Islands │ - │ Flemish Region │ - │ Brussels-Capital Region │ - │ Wallonia │ - │ Federation of Bosnia and Herzegovina │ - └──────────────────────────────────────────────────────────────────┘ - -regionToCountry(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Converts a region to a country. In every other way, this function is the same as 'regionToCity'. -Example: ``regionToCountry(toUInt32(213)) = 225`` converts ``Moscow (213)`` to ``Russia (225)``. - -regionToContinent(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Converts a region to a continent. In every other way, this function is the same as 'regionToCity'. -Example: ``regionToContinent(toUInt32(213)) = 10001`` converts Moscow (213) to Eurasia (10001). - -regionToPopulation(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Gets the population for a region. -The population can be recorded in files with the geobase. See the section "External dictionaries". -If the population is not recorded for the region, it returns 0. -In the Yandex geobase, the population might be recorded for child regions, but not for parent regions.. - -regionIn(lhs, rhs[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Checks whether a 'lhs' region belongs to a 'rhs' region. Returns a UInt8 number equal to 1 if it belongs, or 0 if it doesn't belong. -The relationship is reflexive - any region also belongs to itself. - -regionHierarchy(id[, geobase]) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Accepts a UInt32 number - the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. -Example: ``regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]``. - -regionToName(id[, lang]) -~~~~~~~~~~~~~~~~~~~~~~~~ -Accepts a UInt32 number - the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language 'ru' is used. If the language is not supported, an exception is thrown. Returns a string - the name of the region in the corresponding language. If the region with the specified ID doesn't exist, an empty string is returned. - -``ua`` and ``uk`` mean the same thing - Ukrainian. diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md new file mode 100644 index 00000000000..49265d5da85 --- /dev/null +++ b/docs/en/getting_started/example_datasets/amplab_benchmark.md @@ -0,0 +1,121 @@ +# AMPLab Big Data Benchmark + +See + +Sign up for a free account at . You will need a credit card, email and phone number.Get a new access key at + +Run the following in the console: + +```bash +sudo apt-get install s3cmd +mkdir tiny; cd tiny; +s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . +cd .. +mkdir 1node; cd 1node; +s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . +cd .. +mkdir 5nodes; cd 5nodes; +s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . +cd .. +``` + +Run the following ClickHouse queries: + +```sql +CREATE TABLE rankings_tiny +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_tiny +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); + +CREATE TABLE rankings_1node +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_1node +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); + +CREATE TABLE rankings_5nodes_on_single +( + pageURL String, + pageRank UInt32, + avgDuration UInt32 +) ENGINE = Log; + +CREATE TABLE uservisits_5nodes_on_single +( + sourceIP String, + destinationURL String, + visitDate Date, + adRevenue Float32, + UserAgent String, + cCode FixedString(3), + lCode FixedString(6), + searchWord String, + duration UInt32 +) ENGINE = MergeTree(visitDate, visitDate, 8192); +``` + +Go back to the console: + +```bash +for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done +for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done +for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done +for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done +for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done +for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done +``` + +Queries for obtaining data samples: + +```sql +SELECT pageURL, pageRank FROM rankings_1node WHERE pageRank > 1000 + +SELECT substring(sourceIP, 1, 8), sum(adRevenue) FROM uservisits_1node GROUP BY substring(sourceIP, 1, 8) + +SELECT + sourceIP, + sum(adRevenue) AS totalRevenue, + avg(pageRank) AS pageRank +FROM rankings_1node ALL INNER JOIN +( + SELECT + sourceIP, + destinationURL AS pageURL, + adRevenue + FROM uservisits_1node + WHERE (visitDate > '1980-01-01') AND (visitDate < '1980-04-01') +) USING pageURL +GROUP BY sourceIP +ORDER BY totalRevenue DESC +LIMIT 1 +``` + diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.rst b/docs/en/getting_started/example_datasets/amplab_benchmark.rst deleted file mode 100644 index 610a8c12253..00000000000 --- a/docs/en/getting_started/example_datasets/amplab_benchmark.rst +++ /dev/null @@ -1,124 +0,0 @@ -AMPLab Big Data Benchmark -------------------------- - -See: https://amplab.cs.berkeley.edu/benchmark/ - -Register free account on https://aws.amazon.com - you will need credit card, email and phone number -Create new access key on https://console.aws.amazon.com/iam/home?nc2=h_m_sc#security_credential - -Execute this in shell: - -.. code-block:: bash - - sudo apt-get install s3cmd - mkdir tiny; cd tiny; - s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . - cd .. - mkdir 1node; cd 1node; - s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . - cd .. - mkdir 5nodes; cd 5nodes; - s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . - cd .. - -Execute these ClickHouse queries: - -.. code-block:: sql - - CREATE TABLE rankings_tiny - ( - pageURL String, - pageRank UInt32, - avgDuration UInt32 - ) ENGINE = Log; - - CREATE TABLE uservisits_tiny - ( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 - ) ENGINE = MergeTree(visitDate, visitDate, 8192); - - CREATE TABLE rankings_1node - ( - pageURL String, - pageRank UInt32, - avgDuration UInt32 - ) ENGINE = Log; - - CREATE TABLE uservisits_1node - ( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 - ) ENGINE = MergeTree(visitDate, visitDate, 8192); - - CREATE TABLE rankings_5nodes_on_single - ( - pageURL String, - pageRank UInt32, - avgDuration UInt32 - ) ENGINE = Log; - - CREATE TABLE uservisits_5nodes_on_single - ( - sourceIP String, - destinationURL String, - visitDate Date, - adRevenue Float32, - UserAgent String, - cCode FixedString(3), - lCode FixedString(6), - searchWord String, - duration UInt32 - ) ENGINE = MergeTree(visitDate, visitDate, 8192); - - -Back into shell: - -.. code-block:: bash - - for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done - for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done - for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done - for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done - for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done - for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done - - -Data retrieval queries: - -.. code-block:: sql - - SELECT pageURL, pageRank FROM rankings_1node WHERE pageRank > 1000 - - SELECT substring(sourceIP, 1, 8), sum(adRevenue) FROM uservisits_1node GROUP BY substring(sourceIP, 1, 8) - - SELECT - sourceIP, - sum(adRevenue) AS totalRevenue, - avg(pageRank) AS pageRank - FROM rankings_1node ALL INNER JOIN - ( - SELECT - sourceIP, - destinationURL AS pageURL, - adRevenue - FROM uservisits_1node - WHERE (visitDate > '1980-01-01') AND (visitDate < '1980-04-01') - ) USING pageURL - GROUP BY sourceIP - ORDER BY totalRevenue DESC - LIMIT 1 diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md new file mode 100644 index 00000000000..9b59d6e5f3d --- /dev/null +++ b/docs/en/getting_started/example_datasets/criteo.md @@ -0,0 +1,71 @@ +# Terabyte of click logs from Criteo + +Download the data from + +Create a table to import the log to: + +```sql +CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log +``` + +Download the data: + +```bash +for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done +``` + +Create a table for the converted data: + +```sql +CREATE TABLE criteo +( + date Date, + clicked UInt8, + int1 Int32, + int2 Int32, + int3 Int32, + int4 Int32, + int5 Int32, + int6 Int32, + int7 Int32, + int8 Int32, + int9 Int32, + int10 Int32, + int11 Int32, + int12 Int32, + int13 Int32, + icat1 UInt32, + icat2 UInt32, + icat3 UInt32, + icat4 UInt32, + icat5 UInt32, + icat6 UInt32, + icat7 UInt32, + icat8 UInt32, + icat9 UInt32, + icat10 UInt32, + icat11 UInt32, + icat12 UInt32, + icat13 UInt32, + icat14 UInt32, + icat15 UInt32, + icat16 UInt32, + icat17 UInt32, + icat18 UInt32, + icat19 UInt32, + icat20 UInt32, + icat21 UInt32, + icat22 UInt32, + icat23 UInt32, + icat24 UInt32, + icat25 UInt32, + icat26 UInt32 +) ENGINE = MergeTree(date, intHash32(icat1), (date, intHash32(icat1)), 8192) +``` + +Transform data from the raw log and put it in the second table: + +```sql +INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;DROP TABLE criteo_log; +``` + diff --git a/docs/en/getting_started/example_datasets/criteo.rst b/docs/en/getting_started/example_datasets/criteo.rst deleted file mode 100644 index 5c363596125..00000000000 --- a/docs/en/getting_started/example_datasets/criteo.rst +++ /dev/null @@ -1,73 +0,0 @@ -Criteo Terabyte Click Logs --------------------------- - -Download all data from http://labs.criteo.com/downloads/download-terabyte-click-logs/ - -Create log table: - -.. code-block:: sql - - CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log - -Load data: - -.. code-block:: bash - - for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done - -Create table for converted data: - -.. code-block:: sql - - CREATE TABLE criteo - ( - date Date, - clicked UInt8, - int1 Int32, - int2 Int32, - int3 Int32, - int4 Int32, - int5 Int32, - int6 Int32, - int7 Int32, - int8 Int32, - int9 Int32, - int10 Int32, - int11 Int32, - int12 Int32, - int13 Int32, - icat1 UInt32, - icat2 UInt32, - icat3 UInt32, - icat4 UInt32, - icat5 UInt32, - icat6 UInt32, - icat7 UInt32, - icat8 UInt32, - icat9 UInt32, - icat10 UInt32, - icat11 UInt32, - icat12 UInt32, - icat13 UInt32, - icat14 UInt32, - icat15 UInt32, - icat16 UInt32, - icat17 UInt32, - icat18 UInt32, - icat19 UInt32, - icat20 UInt32, - icat21 UInt32, - icat22 UInt32, - icat23 UInt32, - icat24 UInt32, - icat25 UInt32, - icat26 UInt32 - ) ENGINE = MergeTree(date, intHash32(icat1), (date, intHash32(icat1)), 8192) - -Parse the data from raw log and put it into second table: - -.. code-block:: sql - - INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; - - DROP TABLE criteo_log; diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md new file mode 100644 index 00000000000..a48d0730b8c --- /dev/null +++ b/docs/en/getting_started/example_datasets/nyc_taxi.md @@ -0,0 +1,371 @@ +# Data about New York taxis + +## How to import raw data + +See +и +для описания набора данных и инструкций по загрузке. + +Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel). +Some of the files might not download fully. Check the file sizes and re-download any that seem doubtful. + +Some of the files might contain invalid rows. You can fix them as follows: + +```bash +sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_ +sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-03.csv > data/yellow_tripdata_2010-03.csv_ +mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv +mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv +``` + +Then the data must be pre-processed in PostgreSQL. This will create selections of points in the polygons (to match points on the map with the boroughs of New York City) and combine all the data into a single denormalized flat table by using a JOIN. To do this, you will need to install PostgreSQL with PostGIS support. + +Be careful when running `initialize_database.sh` and manually re-check that all the tables were created correctly. + +It takes about 20-30 minutes to process each month's worth of data in PostgreSQL, for a total of about 48 hours. + +You can check the number of downloaded rows as follows: + +```text +time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" +## count + 1298979494 +(1 row) + +real 7m9.164s +``` + +(This is slightly more than 1.1 billion rows reported by Mark Litwintschik in a series of blog posts.) + +The data in PostgreSQL uses 370 GB of space. + +Exporting the data from PostgreSQL: + +```sql +COPY +( + SELECT trips.id, + trips.vendor_id, + trips.pickup_datetime, + trips.dropoff_datetime, + trips.store_and_fwd_flag, + trips.rate_code_id, + trips.pickup_longitude, + trips.pickup_latitude, + trips.dropoff_longitude, + trips.dropoff_latitude, + trips.passenger_count, + trips.trip_distance, + trips.fare_amount, + trips.extra, + trips.mta_tax, + trips.tip_amount, + trips.tolls_amount, + trips.ehail_fee, + trips.improvement_surcharge, + trips.total_amount, + trips.payment_type, + trips.trip_type, + trips.pickup, + trips.dropoff, + + cab_types.type cab_type, + + weather.precipitation_tenths_of_mm rain, + weather.snow_depth_mm, + weather.snowfall_mm, + weather.max_temperature_tenths_degrees_celsius max_temp, + weather.min_temperature_tenths_degrees_celsius min_temp, + weather.average_wind_speed_tenths_of_meters_per_second wind, + + pick_up.gid pickup_nyct2010_gid, + pick_up.ctlabel pickup_ctlabel, + pick_up.borocode pickup_borocode, + pick_up.boroname pickup_boroname, + pick_up.ct2010 pickup_ct2010, + pick_up.boroct2010 pickup_boroct2010, + pick_up.cdeligibil pickup_cdeligibil, + pick_up.ntacode pickup_ntacode, + pick_up.ntaname pickup_ntaname, + pick_up.puma pickup_puma, + + drop_off.gid dropoff_nyct2010_gid, + drop_off.ctlabel dropoff_ctlabel, + drop_off.borocode dropoff_borocode, + drop_off.boroname dropoff_boroname, + drop_off.ct2010 dropoff_ct2010, + drop_off.boroct2010 dropoff_boroct2010, + drop_off.cdeligibil dropoff_cdeligibil, + drop_off.ntacode dropoff_ntacode, + drop_off.ntaname dropoff_ntaname, + drop_off.puma dropoff_puma + FROM trips + LEFT JOIN cab_types + ON trips.cab_type_id = cab_types.id + LEFT JOIN central_park_weather_observations_raw weather + ON weather.date = trips.pickup_datetime::date + LEFT JOIN nyct2010 pick_up + ON pick_up.gid = trips.pickup_nyct2010_gid + LEFT JOIN nyct2010 drop_off + ON drop_off.gid = trips.dropoff_nyct2010_gid +) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; +``` + +The data snapshot is created at a speed of about 50 MB per second. While creating the snapshot, PostgreSQL reads from the disk at a speed of about 28 MB per second. +This takes about 5 hours. The resulting TSV file is 590612904969 bytes. + +Create a temporary table in ClickHouse: + +```sql +CREATE TABLE trips +( +trip_id UInt32, +vendor_id String, +pickup_datetime DateTime, +dropoff_datetime Nullable(DateTime), +store_and_fwd_flag Nullable(FixedString(1)), +rate_code_id Nullable(UInt8), +pickup_longitude Nullable(Float64), +pickup_latitude Nullable(Float64), +dropoff_longitude Nullable(Float64), +dropoff_latitude Nullable(Float64), +passenger_count Nullable(UInt8), +trip_distance Nullable(Float64), +fare_amount Nullable(Float32), +extra Nullable(Float32), +mta_tax Nullable(Float32), +tip_amount Nullable(Float32), +tolls_amount Nullable(Float32), +ehail_fee Nullable(Float32), +improvement_surcharge Nullable(Float32), +total_amount Nullable(Float32), +payment_type Nullable(String), +trip_type Nullable(UInt8), +pickup Nullable(String), +dropoff Nullable(String), +cab_type Nullable(String), +precipitation Nullable(UInt8), +snow_depth Nullable(UInt8), +snowfall Nullable(UInt8), +max_temperature Nullable(UInt8), +min_temperature Nullable(UInt8), +average_wind_speed Nullable(UInt8), +pickup_nyct2010_gid Nullable(UInt8), +pickup_ctlabel Nullable(String), +pickup_borocode Nullable(UInt8), +pickup_boroname Nullable(String), +pickup_ct2010 Nullable(String), +pickup_boroct2010 Nullable(String), +pickup_cdeligibil Nullable(FixedString(1)), +pickup_ntacode Nullable(String), +pickup_ntaname Nullable(String), +pickup_puma Nullable(String), +dropoff_nyct2010_gid Nullable(UInt8), +dropoff_ctlabel Nullable(String), +dropoff_borocode Nullable(UInt8), +dropoff_boroname Nullable(String), +dropoff_ct2010 Nullable(String), +dropoff_boroct2010 Nullable(String), +dropoff_cdeligibil Nullable(String), +dropoff_ntacode Nullable(String), +dropoff_ntaname Nullable(String), +dropoff_puma Nullable(String) +) ENGINE = Log; +``` + +It is needed for converting fields to more correct data types and, if possible, to eliminate NULLs. + +```text +time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv + +real 75m56.214s +``` + +Data is read at a speed of 112-140 Mb/second. +Loading data into a Log type table in one stream took 76 minutes. +The data in this table uses 142 GB. + +(Importing data directly from Postgres is also possible using ` COPY ... TO PROGRAM`.) + +Unfortunately, all the fields associated with the weather (precipitation...average_wind_speed) were filled with NULL. Because of this, we will remove them from the final data set. + +To start, we'll create a table on a single server. Later we will make the table distributed. + +Create and populate a summary table: + +```text +CREATE TABLE trips_mergetree +ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) +AS SELECT + +trip_id, +CAST(vendor_id AS Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14)) AS vendor_id, +toDate(pickup_datetime) AS pickup_date, +ifNull(pickup_datetime, toDateTime(0)) AS pickup_datetime, +toDate(dropoff_datetime) AS dropoff_date, +ifNull(dropoff_datetime, toDateTime(0)) AS dropoff_datetime, +assumeNotNull(store_and_fwd_flag) IN ('Y', '1', '2') AS store_and_fwd_flag, +assumeNotNull(rate_code_id) AS rate_code_id, +assumeNotNull(pickup_longitude) AS pickup_longitude, +assumeNotNull(pickup_latitude) AS pickup_latitude, +assumeNotNull(dropoff_longitude) AS dropoff_longitude, +assumeNotNull(dropoff_latitude) AS dropoff_latitude, +assumeNotNull(passenger_count) AS passenger_count, +assumeNotNull(trip_distance) AS trip_distance, +assumeNotNull(fare_amount) AS fare_amount, +assumeNotNull(extra) AS extra, +assumeNotNull(mta_tax) AS mta_tax, +assumeNotNull(tip_amount) AS tip_amount, +assumeNotNull(tolls_amount) AS tolls_amount, +assumeNotNull(ehail_fee) AS ehail_fee, +assumeNotNull(improvement_surcharge) AS improvement_surcharge, +assumeNotNull(total_amount) AS total_amount, +CAST((assumeNotNull(payment_type) AS pt) IN ('CSH', 'CASH', 'Cash', 'CAS', 'Cas', '1') ? 'CSH' : (pt IN ('CRD', 'Credit', 'Cre', 'CRE', 'CREDIT', '2') ? 'CRE' : (pt IN ('NOC', 'No Charge', 'No', '3') ? 'NOC' : (pt IN ('DIS', 'Dispute', 'Dis', '4') ? 'DIS' : 'UNK'))) AS Enum8('CSH' = 1, 'CRE' = 2, 'UNK' = 0, 'NOC' = 3, 'DIS' = 4)) AS payment_type_, +assumeNotNull(trip_type) AS trip_type, +ifNull(toFixedString(unhex(pickup), 25), toFixedString('', 25)) AS pickup, +ifNull(toFixedString(unhex(dropoff), 25), toFixedString('', 25)) AS dropoff, +CAST(assumeNotNull(cab_type) AS Enum8('yellow' = 1, 'green' = 2, 'uber' = 3)) AS cab_type, + +assumeNotNull(pickup_nyct2010_gid) AS pickup_nyct2010_gid, +toFloat32(ifNull(pickup_ctlabel, '0')) AS pickup_ctlabel, +assumeNotNull(pickup_borocode) AS pickup_borocode, +CAST(assumeNotNull(pickup_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS pickup_boroname, +toFixedString(ifNull(pickup_ct2010, '000000'), 6) AS pickup_ct2010, +toFixedString(ifNull(pickup_boroct2010, '0000000'), 7) AS pickup_boroct2010, +CAST(assumeNotNull(ifNull(pickup_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS pickup_cdeligibil, +toFixedString(ifNull(pickup_ntacode, '0000'), 4) AS pickup_ntacode, + +CAST(assumeNotNull(pickup_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS pickup_ntaname, + +toUInt16(ifNull(pickup_puma, '0')) AS pickup_puma, + +assumeNotNull(dropoff_nyct2010_gid) AS dropoff_nyct2010_gid, +toFloat32(ifNull(dropoff_ctlabel, '0')) AS dropoff_ctlabel, +assumeNotNull(dropoff_borocode) AS dropoff_borocode, +CAST(assumeNotNull(dropoff_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS dropoff_boroname, +toFixedString(ifNull(dropoff_ct2010, '000000'), 6) AS dropoff_ct2010, +toFixedString(ifNull(dropoff_boroct2010, '0000000'), 7) AS dropoff_boroct2010, +CAST(assumeNotNull(ifNull(dropoff_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS dropoff_cdeligibil, +toFixedString(ifNull(dropoff_ntacode, '0000'), 4) AS dropoff_ntacode, + +CAST(assumeNotNull(dropoff_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS dropoff_ntaname, + +toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma + +FROM trips +``` + +This takes 3030 seconds at a speed of about 428,000 rows per second. +To load it faster, you can create the table with the `Log` engine instead of `MergeTree`. In this case, the download works faster than 200 seconds. + +The table uses 126 GB of disk space. + +```text +:) SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active + +SELECT formatReadableSize(sum(bytes)) +FROM system.parts +WHERE (table = 'trips_mergetree') AND active + +┌─formatReadableSize(sum(bytes))─┐ +│ 126.18 GiB │ +└────────────────────────────────┘ +``` + +Among other things, you can run the OPTIMIZE query on MergeTree. But it's not required, since everything will be fine without it. + +## Results on single server + +Q1: + +```sql +SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type +``` + +0.490 seconds. + +Q2: + +```sql +SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count +``` + +1.224 seconds. + +Q3: + +```sql +SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year +``` + +2.104 seconds. + +Q4: + +```sql +SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) +FROM trips_mergetree +GROUP BY passenger_count, year, distance +ORDER BY year, count(*) DESC +``` + +3.593 seconds. + +The following server was used: + +Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total, +128 GiB RAM, +8x6 TB HD on hardware RAID-5 + +Execution time is the best of three runsBut starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run. + +Creating a table on three servers: + +On each server: + +```text +CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) +``` + +On the source server: + +```sql +CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) +``` + +The following query redistributes data: + +```sql +INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree +``` + +This takes 2454 seconds. + +On three servers: + +Q1: 0.212 seconds. +Q2: 0.438 seconds. +Q3: 0.733 seconds. +Q4: 1.241 seconds. + +No surprises here, since the queries are scaled linearly. + +We also have results from a cluster of 140 servers: + +Q1: 0.028 sec. +Q2: 0.043 sec. +Q3: 0.051 sec. +Q4: 0.072 sec. + +In this case, the query processing time is determined above all by network latency. +We ran queries using a client located in a Yandex datacenter in Finland on a cluster in Russia, which added about 20 ms of latency. + +## Summary + +```text +nodes Q1 Q2 Q3 Q4 + 1 0.490 1.224 2.104 3.593 + 3 0.212 0.438 0.733 1.241 +140 0.028 0.043 0.051 0.072 +``` + diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.rst b/docs/en/getting_started/example_datasets/nyc_taxi.rst deleted file mode 100644 index 7d5b47f0c09..00000000000 --- a/docs/en/getting_started/example_datasets/nyc_taxi.rst +++ /dev/null @@ -1,387 +0,0 @@ -NYC Taxi Dataset -================ - -How to create dataset from raw data ------------------------------------ - -Look at https://github.com/toddwschneider/nyc-taxi-data -and http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html -for description of the dataset and loading instructions. - -Data will download to ~227 GB of uncompressed CSV files. It takes about one hour on 1 Gbit connection. -(Parallel download from s3.amazonaws.com saturate at least half of one gigabit.) -Some files could be downloaded incompletely. Look at suspicious file sizes and repeat downloading of incomplete files. - -Some files contain broken rows. To correct them, run: - -.. code-block:: bash - - sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_ - sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-03.csv > data/yellow_tripdata_2010-03.csv_ - mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv - mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv - - -Then data must be preprocessed inside PostgreSQL. It will do point-in-polygon lookups (map points to areas of New York), and finally JOIN all data to single denormalized flat table. You must install PostgreSQL with PostGIS support. - -When running ``initialize_database.sh`` script, be careful and check manually, that all tables get created successfully. - -Processing of each month of yellow taxi data in PostgreSQL takes about 20-30 minutes, about 48 hours in total. - -Check exact amount of loaded rows: - -.. code-block:: text - - time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" - count - ------------ - 1298979494 - (1 row) - - real 7m9.164s - -(this is slightly more than 1.1 billion rows reported by Mark Litwintschik in a series of blog posts) - -Data in PostgreSQL takes 370 GB (346 GiB). - -Export data from PostgreSQL: - -.. code-block:: sql - - COPY - ( - SELECT trips.id, - trips.vendor_id, - trips.pickup_datetime, - trips.dropoff_datetime, - trips.store_and_fwd_flag, - trips.rate_code_id, - trips.pickup_longitude, - trips.pickup_latitude, - trips.dropoff_longitude, - trips.dropoff_latitude, - trips.passenger_count, - trips.trip_distance, - trips.fare_amount, - trips.extra, - trips.mta_tax, - trips.tip_amount, - trips.tolls_amount, - trips.ehail_fee, - trips.improvement_surcharge, - trips.total_amount, - trips.payment_type, - trips.trip_type, - trips.pickup, - trips.dropoff, - - cab_types.type cab_type, - - weather.precipitation_tenths_of_mm rain, - weather.snow_depth_mm, - weather.snowfall_mm, - weather.max_temperature_tenths_degrees_celsius max_temp, - weather.min_temperature_tenths_degrees_celsius min_temp, - weather.average_wind_speed_tenths_of_meters_per_second wind, - - pick_up.gid pickup_nyct2010_gid, - pick_up.ctlabel pickup_ctlabel, - pick_up.borocode pickup_borocode, - pick_up.boroname pickup_boroname, - pick_up.ct2010 pickup_ct2010, - pick_up.boroct2010 pickup_boroct2010, - pick_up.cdeligibil pickup_cdeligibil, - pick_up.ntacode pickup_ntacode, - pick_up.ntaname pickup_ntaname, - pick_up.puma pickup_puma, - - drop_off.gid dropoff_nyct2010_gid, - drop_off.ctlabel dropoff_ctlabel, - drop_off.borocode dropoff_borocode, - drop_off.boroname dropoff_boroname, - drop_off.ct2010 dropoff_ct2010, - drop_off.boroct2010 dropoff_boroct2010, - drop_off.cdeligibil dropoff_cdeligibil, - drop_off.ntacode dropoff_ntacode, - drop_off.ntaname dropoff_ntaname, - drop_off.puma dropoff_puma - FROM trips - LEFT JOIN cab_types - ON trips.cab_type_id = cab_types.id - LEFT JOIN central_park_weather_observations_raw weather - ON weather.date = trips.pickup_datetime::date - LEFT JOIN nyct2010 pick_up - ON pick_up.gid = trips.pickup_nyct2010_gid - LEFT JOIN nyct2010 drop_off - ON drop_off.gid = trips.dropoff_nyct2010_gid - ) TO '/opt/milovidov/nyc-taxi-data/trips.tsv'; - - -Dump is created at speed about 50 MB/sec. While creating dump, PostgreSQL reads data from disk at about 28 MB/sec. -It takes about 5 hours. Resulting tsv file is 590612904969 bytes. - -Create temporary table in ClickHouse: - -.. code-block:: sql - - CREATE TABLE trips - ( - trip_id UInt32, - vendor_id String, - pickup_datetime DateTime, - dropoff_datetime Nullable(DateTime), - store_and_fwd_flag Nullable(FixedString(1)), - rate_code_id Nullable(UInt8), - pickup_longitude Nullable(Float64), - pickup_latitude Nullable(Float64), - dropoff_longitude Nullable(Float64), - dropoff_latitude Nullable(Float64), - passenger_count Nullable(UInt8), - trip_distance Nullable(Float64), - fare_amount Nullable(Float32), - extra Nullable(Float32), - mta_tax Nullable(Float32), - tip_amount Nullable(Float32), - tolls_amount Nullable(Float32), - ehail_fee Nullable(Float32), - improvement_surcharge Nullable(Float32), - total_amount Nullable(Float32), - payment_type Nullable(String), - trip_type Nullable(UInt8), - pickup Nullable(String), - dropoff Nullable(String), - cab_type Nullable(String), - precipitation Nullable(UInt8), - snow_depth Nullable(UInt8), - snowfall Nullable(UInt8), - max_temperature Nullable(UInt8), - min_temperature Nullable(UInt8), - average_wind_speed Nullable(UInt8), - pickup_nyct2010_gid Nullable(UInt8), - pickup_ctlabel Nullable(String), - pickup_borocode Nullable(UInt8), - pickup_boroname Nullable(String), - pickup_ct2010 Nullable(String), - pickup_boroct2010 Nullable(String), - pickup_cdeligibil Nullable(FixedString(1)), - pickup_ntacode Nullable(String), - pickup_ntaname Nullable(String), - pickup_puma Nullable(String), - dropoff_nyct2010_gid Nullable(UInt8), - dropoff_ctlabel Nullable(String), - dropoff_borocode Nullable(UInt8), - dropoff_boroname Nullable(String), - dropoff_ct2010 Nullable(String), - dropoff_boroct2010 Nullable(String), - dropoff_cdeligibil Nullable(String), - dropoff_ntacode Nullable(String), - dropoff_ntaname Nullable(String), - dropoff_puma Nullable(String) - ) ENGINE = Log; - - -This table is needed to select better data types for fields, and if possible, get rid of NULLs. - -.. code-block:: text - - time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv - - real 75m56.214s - - -Data is read at 112-140 MB/sec. -Loading data into Log table in single thread took 76 minutes. -Data inside Log table takes 142 GB. - -(You could also import data directly from Postgres, using ``COPY ... TO PROGRAM``.) - -Sadly, all weather-related fields (precipitation...average_wind_speed) are NULLs. We will omit them from final dataset. - -For the first time, we will create a table on single server. Later we will distribute the table. -Create and fill final table: - -.. code-block:: text - - CREATE TABLE trips_mergetree - ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) - AS SELECT - - trip_id, - CAST(vendor_id AS Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14)) AS vendor_id, - toDate(pickup_datetime) AS pickup_date, - ifNull(pickup_datetime, toDateTime(0)) AS pickup_datetime, - toDate(dropoff_datetime) AS dropoff_date, - ifNull(dropoff_datetime, toDateTime(0)) AS dropoff_datetime, - assumeNotNull(store_and_fwd_flag) IN ('Y', '1', '2') AS store_and_fwd_flag, - assumeNotNull(rate_code_id) AS rate_code_id, - assumeNotNull(pickup_longitude) AS pickup_longitude, - assumeNotNull(pickup_latitude) AS pickup_latitude, - assumeNotNull(dropoff_longitude) AS dropoff_longitude, - assumeNotNull(dropoff_latitude) AS dropoff_latitude, - assumeNotNull(passenger_count) AS passenger_count, - assumeNotNull(trip_distance) AS trip_distance, - assumeNotNull(fare_amount) AS fare_amount, - assumeNotNull(extra) AS extra, - assumeNotNull(mta_tax) AS mta_tax, - assumeNotNull(tip_amount) AS tip_amount, - assumeNotNull(tolls_amount) AS tolls_amount, - assumeNotNull(ehail_fee) AS ehail_fee, - assumeNotNull(improvement_surcharge) AS improvement_surcharge, - assumeNotNull(total_amount) AS total_amount, - CAST((assumeNotNull(payment_type) AS pt) IN ('CSH', 'CASH', 'Cash', 'CAS', 'Cas', '1') ? 'CSH' : (pt IN ('CRD', 'Credit', 'Cre', 'CRE', 'CREDIT', '2') ? 'CRE' : (pt IN ('NOC', 'No Charge', 'No', '3') ? 'NOC' : (pt IN ('DIS', 'Dispute', 'Dis', '4') ? 'DIS' : 'UNK'))) AS Enum8('CSH' = 1, 'CRE' = 2, 'UNK' = 0, 'NOC' = 3, 'DIS' = 4)) AS payment_type_, - assumeNotNull(trip_type) AS trip_type, - ifNull(toFixedString(unhex(pickup), 25), toFixedString('', 25)) AS pickup, - ifNull(toFixedString(unhex(dropoff), 25), toFixedString('', 25)) AS dropoff, - CAST(assumeNotNull(cab_type) AS Enum8('yellow' = 1, 'green' = 2, 'uber' = 3)) AS cab_type, - - assumeNotNull(pickup_nyct2010_gid) AS pickup_nyct2010_gid, - toFloat32(ifNull(pickup_ctlabel, '0')) AS pickup_ctlabel, - assumeNotNull(pickup_borocode) AS pickup_borocode, - CAST(assumeNotNull(pickup_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS pickup_boroname, - toFixedString(ifNull(pickup_ct2010, '000000'), 6) AS pickup_ct2010, - toFixedString(ifNull(pickup_boroct2010, '0000000'), 7) AS pickup_boroct2010, - CAST(assumeNotNull(ifNull(pickup_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS pickup_cdeligibil, - toFixedString(ifNull(pickup_ntacode, '0000'), 4) AS pickup_ntacode, - - CAST(assumeNotNull(pickup_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS pickup_ntaname, - - toUInt16(ifNull(pickup_puma, '0')) AS pickup_puma, - - assumeNotNull(dropoff_nyct2010_gid) AS dropoff_nyct2010_gid, - toFloat32(ifNull(dropoff_ctlabel, '0')) AS dropoff_ctlabel, - assumeNotNull(dropoff_borocode) AS dropoff_borocode, - CAST(assumeNotNull(dropoff_boroname) AS Enum8('Manhattan' = 1, 'Queens' = 4, 'Brooklyn' = 3, '' = 0, 'Bronx' = 2, 'Staten Island' = 5)) AS dropoff_boroname, - toFixedString(ifNull(dropoff_ct2010, '000000'), 6) AS dropoff_ct2010, - toFixedString(ifNull(dropoff_boroct2010, '0000000'), 7) AS dropoff_boroct2010, - CAST(assumeNotNull(ifNull(dropoff_cdeligibil, ' ')) AS Enum8(' ' = 0, 'E' = 1, 'I' = 2)) AS dropoff_cdeligibil, - toFixedString(ifNull(dropoff_ntacode, '0000'), 4) AS dropoff_ntacode, - - CAST(assumeNotNull(dropoff_ntaname) AS Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195)) AS dropoff_ntaname, - - toUInt16(ifNull(dropoff_puma, '0')) AS dropoff_puma - - FROM trips - - -This is done in 3030 sec at about 428000 rows/sec. -If you want faster load time, you may create table with ``Log`` engine instead of ``MergeTree``. In that case, load is under 200 sec. - -Table has used 126 GiB of disk space. - -.. code-block:: text - - :) SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active - - SELECT formatReadableSize(sum(bytes)) - FROM system.parts - WHERE (table = 'trips_mergetree') AND active - - ┌─formatReadableSize(sum(bytes))─┐ - │ 126.18 GiB │ - └────────────────────────────────┘ - - -BTW, you could run OPTIMIZE for MergeTree table. But this is not necessary, everything will be fine. - - -Results on single server ------------------------- - -Q1: - -.. code-block:: sql - - SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type - -0.490 sec. - -Q2: - -.. code-block:: sql - - SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count - -1.224 sec. - -Q3: - -.. code-block:: sql - - SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year - -2.104 sec. - -Q4: - -.. code-block:: sql - - SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) - FROM trips_mergetree - GROUP BY passenger_count, year, distance - ORDER BY year, count(*) DESC - -3.593 sec. - -The server is: - -Two-socket Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, total 16 physical cores, -128 GiB RAM, -8x6 TB HDD in software RAID-5 - -Query time is best of three runs. -Actually, starting from second run, the query reads data from OS page cache. No further caching happens: data is read and processed in each run. - -Create table on cluster with 3 servers: - -On each server: - -.. code-block:: text - - CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) - - -On source server: - -.. code-block:: sql - - CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) - - -This will distribute the data: - -.. code-block:: sql - - INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree - - -This takes 2454 sec. - -On three servers: - -Q1: 0.212 sec. -Q2: 0.438 sec. -Q3: 0.733 sec. -Q4: 1.241 sec. - -No surprise, as queries are linearly scalable. - - -Also there are results for 140-node cluster: - -Q1: 0.028 sec. -Q2: 0.043 sec. -Q3: 0.051 sec. -Q4: 0.072 sec. - -In that case, query execution speed is dominated by latency. -We do queries from client located in Yandex datacenter in Mäntsälä (Finland) to cluster somewhere in Russia, that adds at least 20 ms of latency. - -Summary -------- - -.. code-block:: text - - nodes Q1 Q2 Q3 Q4 - 1 0.490 1.224 2.104 3.593 - 3 0.212 0.438 0.733 1.241 - 140 0.028 0.043 0.051 0.072 diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md new file mode 100644 index 00000000000..574e195e6b5 --- /dev/null +++ b/docs/en/getting_started/example_datasets/ontime.md @@ -0,0 +1,319 @@ + + +# OnTime + +This performance test was created by Vadim Tkachenko. See: + +- +- +- +- +- +- + +Downloading data: + +```bash +for s in `seq 1987 2017` +do +for m in `seq 1 12` +do +wget http://transtats.bts.gov/PREZIP/On_Time_On_Time_Performance_${s}_${m}.zip +done +done +``` + +(from ) + +Creating a table: + +```sql +CREATE TABLE `ontime` ( + `Year` UInt16, + `Quarter` UInt8, + `Month` UInt8, + `DayofMonth` UInt8, + `DayOfWeek` UInt8, + `FlightDate` Date, + `UniqueCarrier` FixedString(7), + `AirlineID` Int32, + `Carrier` FixedString(2), + `TailNum` String, + `FlightNum` String, + `OriginAirportID` Int32, + `OriginAirportSeqID` Int32, + `OriginCityMarketID` Int32, + `Origin` FixedString(5), + `OriginCityName` String, + `OriginState` FixedString(2), + `OriginStateFips` String, + `OriginStateName` String, + `OriginWac` Int32, + `DestAirportID` Int32, + `DestAirportSeqID` Int32, + `DestCityMarketID` Int32, + `Dest` FixedString(5), + `DestCityName` String, + `DestState` FixedString(2), + `DestStateFips` String, + `DestStateName` String, + `DestWac` Int32, + `CRSDepTime` Int32, + `DepTime` Int32, + `DepDelay` Int32, + `DepDelayMinutes` Int32, + `DepDel15` Int32, + `DepartureDelayGroups` String, + `DepTimeBlk` String, + `TaxiOut` Int32, + `WheelsOff` Int32, + `WheelsOn` Int32, + `TaxiIn` Int32, + `CRSArrTime` Int32, + `ArrTime` Int32, + `ArrDelay` Int32, + `ArrDelayMinutes` Int32, + `ArrDel15` Int32, + `ArrivalDelayGroups` Int32, + `ArrTimeBlk` String, + `Cancelled` UInt8, + `CancellationCode` FixedString(1), + `Diverted` UInt8, + `CRSElapsedTime` Int32, + `ActualElapsedTime` Int32, + `AirTime` Int32, + `Flights` Int32, + `Distance` Int32, + `DistanceGroup` UInt8, + `CarrierDelay` Int32, + `WeatherDelay` Int32, + `NASDelay` Int32, + `SecurityDelay` Int32, + `LateAircraftDelay` Int32, + `FirstDepTime` String, + `TotalAddGTime` String, + `LongestAddGTime` String, + `DivAirportLandings` String, + `DivReachedDest` String, + `DivActualElapsedTime` String, + `DivArrDelay` String, + `DivDistance` String, + `Div1Airport` String, + `Div1AirportID` Int32, + `Div1AirportSeqID` Int32, + `Div1WheelsOn` String, + `Div1TotalGTime` String, + `Div1LongestGTime` String, + `Div1WheelsOff` String, + `Div1TailNum` String, + `Div2Airport` String, + `Div2AirportID` Int32, + `Div2AirportSeqID` Int32, + `Div2WheelsOn` String, + `Div2TotalGTime` String, + `Div2LongestGTime` String, + `Div2WheelsOff` String, + `Div2TailNum` String, + `Div3Airport` String, + `Div3AirportID` Int32, + `Div3AirportSeqID` Int32, + `Div3WheelsOn` String, + `Div3TotalGTime` String, + `Div3LongestGTime` String, + `Div3WheelsOff` String, + `Div3TailNum` String, + `Div4Airport` String, + `Div4AirportID` Int32, + `Div4AirportSeqID` Int32, + `Div4WheelsOn` String, + `Div4TotalGTime` String, + `Div4LongestGTime` String, + `Div4WheelsOff` String, + `Div4TailNum` String, + `Div5Airport` String, + `Div5AirportID` Int32, + `Div5AirportSeqID` Int32, + `Div5WheelsOn` String, + `Div5TotalGTime` String, + `Div5LongestGTime` String, + `Div5WheelsOff` String, + `Div5TailNum` String +) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192) +``` + +Loading data: + +```bash +for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done +``` + +Queries: + +Q0. + +```sql +select avg(c1) from (select Year, Month, count(*) as c1 from ontime group by Year, Month); +``` + +Q1. The number of flights per day from the year 2000 to 2008 + +```sql +SELECT DayOfWeek, count(*) AS c FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC; +``` + +Q2. The number of flights delayed by more than 10 minutes, grouped by the day of the week, for 2000-2008 + +```sql +SELECT DayOfWeek, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC +``` + +Q3. The number of delays by airport for 2000-2008 + +```sql +SELECT Origin, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY Origin ORDER BY c DESC LIMIT 10 +``` + +Q4. The number of delays by carrier for 2007 + +```sql +SELECT Carrier, count(*) FROM ontime WHERE DepDelay>10 AND Year = 2007 GROUP BY Carrier ORDER BY count(*) DESC +``` + +Q5. The percentage of delays by carrier for 2007 + +```sql +SELECT Carrier, c, c2, c*1000/c2 as c3 +FROM +( + SELECT + Carrier, + count(*) AS c + FROM ontime + WHERE DepDelay>10 + AND Year=2007 + GROUP BY Carrier +) +ANY INNER JOIN +( + SELECT + Carrier, + count(*) AS c2 + FROM ontime + WHERE Year=2007 + GROUP BY Carrier +) USING Carrier +ORDER BY c3 DESC; +``` + +Better version of the same query: + +```sql +SELECT Carrier, avg(DepDelay > 10) * 1000 AS c3 FROM ontime WHERE Year = 2007 GROUP BY Carrier ORDER BY Carrier +``` + +Q6. The previous request for a broader range of years, 2000-2008 + +```sql +SELECT Carrier, c, c2, c*1000/c2 as c3 +FROM +( + SELECT + Carrier, + count(*) AS c + FROM ontime + WHERE DepDelay>10 + AND Year >= 2000 AND Year <= 2008 + GROUP BY Carrier +) +ANY INNER JOIN +( + SELECT + Carrier, + count(*) AS c2 + FROM ontime + WHERE Year >= 2000 AND Year <= 2008 + GROUP BY Carrier +) USING Carrier +ORDER BY c3 DESC; +``` + +Better version of the same query: + +```sql +SELECT Carrier, avg(DepDelay > 10) * 1000 AS c3 FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY Carrier ORDER BY Carrier +``` + +Q7. Percentage of flights delayed for more than 10 minutes, by year + +```sql +SELECT Year, c1/c2 +FROM +( + select + Year, + count(*)*1000 as c1 + from ontime + WHERE DepDelay>10 + GROUP BY Year +) +ANY INNER JOIN +( + select + Year, + count(*) as c2 + from ontime + GROUP BY Year +) USING (Year) +ORDER BY Year +``` + +Better version of the same query: + +```sql +SELECT Year, avg(DepDelay > 10) FROM ontime GROUP BY Year ORDER BY Year +``` + +Q8. The most popular destinations by the number of directly connected cities for various year ranges + +```sql +SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime WHERE Year >= 2000 and Year <= 2010 GROUP BY DestCityName ORDER BY u DESC LIMIT 10; +``` + +Q9. + +```sql +select Year, count(*) as c1 from ontime group by Year; +``` + +Q10. + +```sql +select + min(Year), max(Year), Carrier, count(*) as cnt, + sum(ArrDelayMinutes>30) as flights_delayed, + round(sum(ArrDelayMinutes>30)/count(*),2) as rate +FROM ontime +WHERE + DayOfWeek not in (6,7) and OriginState not in ('AK', 'HI', 'PR', 'VI') + and DestState not in ('AK', 'HI', 'PR', 'VI') + and FlightDate < '2010-01-01' +GROUP by Carrier +HAVING cnt > 100000 and max(Year) > 1990 +ORDER by rate DESC +LIMIT 1000; +``` + +Bonus: + +```sql +SELECT avg(cnt) FROM (SELECT Year,Month,count(*) AS cnt FROM ontime WHERE DepDel15=1 GROUP BY Year,Month) + +select avg(c1) from (select Year,Month,count(*) as c1 from ontime group by Year,Month) + +SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime GROUP BY DestCityName ORDER BY u DESC LIMIT 10; + +SELECT OriginCityName, DestCityName, count() AS c FROM ontime GROUP BY OriginCityName, DestCityName ORDER BY c DESC LIMIT 10; + +SELECT OriginCityName, count() AS c FROM ontime GROUP BY OriginCityName ORDER BY c DESC LIMIT 10; +``` + diff --git a/docs/en/getting_started/example_datasets/ontime.rst b/docs/en/getting_started/example_datasets/ontime.rst deleted file mode 100644 index 386d0c12a51..00000000000 --- a/docs/en/getting_started/example_datasets/ontime.rst +++ /dev/null @@ -1,318 +0,0 @@ -OnTime -====== - -This benchmark was created by Vadim Tkachenko, see: - -* https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ -* https://www.percona.com/blog/2009/10/26/air-traffic-queries-in-luciddb/ -* https://www.percona.com/blog/2009/11/02/air-traffic-queries-in-infinidb-early-alpha/ -* https://www.percona.com/blog/2014/04/21/using-apache-hadoop-and-impala-together-with-mysql-for-data-analysis/ -* https://www.percona.com/blog/2016/01/07/apache-spark-with-air-ontime-performance-data/ -* http://nickmakos.blogspot.ru/2012/08/analyzing-air-traffic-performance-with.html - -Download the data: - -.. code-block:: bash - - for s in `seq 1987 2017` - do - for m in `seq 1 12` - do - wget http://transtats.bts.gov/PREZIP/On_Time_On_Time_Performance_${s}_${m}.zip - done - done - -(from https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh ) - -Create table: - -.. code-block:: sql - - CREATE TABLE `ontime` ( - `Year` UInt16, - `Quarter` UInt8, - `Month` UInt8, - `DayofMonth` UInt8, - `DayOfWeek` UInt8, - `FlightDate` Date, - `UniqueCarrier` FixedString(7), - `AirlineID` Int32, - `Carrier` FixedString(2), - `TailNum` String, - `FlightNum` String, - `OriginAirportID` Int32, - `OriginAirportSeqID` Int32, - `OriginCityMarketID` Int32, - `Origin` FixedString(5), - `OriginCityName` String, - `OriginState` FixedString(2), - `OriginStateFips` String, - `OriginStateName` String, - `OriginWac` Int32, - `DestAirportID` Int32, - `DestAirportSeqID` Int32, - `DestCityMarketID` Int32, - `Dest` FixedString(5), - `DestCityName` String, - `DestState` FixedString(2), - `DestStateFips` String, - `DestStateName` String, - `DestWac` Int32, - `CRSDepTime` Int32, - `DepTime` Int32, - `DepDelay` Int32, - `DepDelayMinutes` Int32, - `DepDel15` Int32, - `DepartureDelayGroups` String, - `DepTimeBlk` String, - `TaxiOut` Int32, - `WheelsOff` Int32, - `WheelsOn` Int32, - `TaxiIn` Int32, - `CRSArrTime` Int32, - `ArrTime` Int32, - `ArrDelay` Int32, - `ArrDelayMinutes` Int32, - `ArrDel15` Int32, - `ArrivalDelayGroups` Int32, - `ArrTimeBlk` String, - `Cancelled` UInt8, - `CancellationCode` FixedString(1), - `Diverted` UInt8, - `CRSElapsedTime` Int32, - `ActualElapsedTime` Int32, - `AirTime` Int32, - `Flights` Int32, - `Distance` Int32, - `DistanceGroup` UInt8, - `CarrierDelay` Int32, - `WeatherDelay` Int32, - `NASDelay` Int32, - `SecurityDelay` Int32, - `LateAircraftDelay` Int32, - `FirstDepTime` String, - `TotalAddGTime` String, - `LongestAddGTime` String, - `DivAirportLandings` String, - `DivReachedDest` String, - `DivActualElapsedTime` String, - `DivArrDelay` String, - `DivDistance` String, - `Div1Airport` String, - `Div1AirportID` Int32, - `Div1AirportSeqID` Int32, - `Div1WheelsOn` String, - `Div1TotalGTime` String, - `Div1LongestGTime` String, - `Div1WheelsOff` String, - `Div1TailNum` String, - `Div2Airport` String, - `Div2AirportID` Int32, - `Div2AirportSeqID` Int32, - `Div2WheelsOn` String, - `Div2TotalGTime` String, - `Div2LongestGTime` String, - `Div2WheelsOff` String, - `Div2TailNum` String, - `Div3Airport` String, - `Div3AirportID` Int32, - `Div3AirportSeqID` Int32, - `Div3WheelsOn` String, - `Div3TotalGTime` String, - `Div3LongestGTime` String, - `Div3WheelsOff` String, - `Div3TailNum` String, - `Div4Airport` String, - `Div4AirportID` Int32, - `Div4AirportSeqID` Int32, - `Div4WheelsOn` String, - `Div4TotalGTime` String, - `Div4LongestGTime` String, - `Div4WheelsOff` String, - `Div4TailNum` String, - `Div5Airport` String, - `Div5AirportID` Int32, - `Div5AirportSeqID` Int32, - `Div5WheelsOn` String, - `Div5TotalGTime` String, - `Div5LongestGTime` String, - `Div5WheelsOff` String, - `Div5TailNum` String - ) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192) - -Load the data: - -.. code-block:: bash - - for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done - -Queries: - - -Q0. - -.. code-block:: sql - - select avg(c1) from (select Year, Month, count(*) as c1 from ontime group by Year, Month); - -Q1. Count flights per day from 2000 to 2008 years - -.. code-block:: sql - - SELECT DayOfWeek, count(*) AS c FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC; - -Q2. Count of flights delayed more than 10min per day of week for 2000-2008 years - -.. code-block:: sql - - SELECT DayOfWeek, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC - -Q3. Count of delays per airport for years 2000-2008 - -.. code-block:: sql - - SELECT Origin, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY Origin ORDER BY c DESC LIMIT 10 - -Q4. Count of delays per Carrier for 2007 year - -.. code-block:: sql - - SELECT Carrier, count(*) FROM ontime WHERE DepDelay>10 AND Year = 2007 GROUP BY Carrier ORDER BY count(*) DESC - -Q5. Percentage of delays for each carrier for 2007 year. - -.. code-block:: sql - - SELECT Carrier, c, c2, c*1000/c2 as c3 - FROM - ( - SELECT - Carrier, - count(*) AS c - FROM ontime - WHERE DepDelay>10 - AND Year=2007 - GROUP BY Carrier - ) - ANY INNER JOIN - ( - SELECT - Carrier, - count(*) AS c2 - FROM ontime - WHERE Year=2007 - GROUP BY Carrier - ) USING Carrier - ORDER BY c3 DESC; - -More optimal version of same query: - -.. code-block:: sql - - SELECT Carrier, avg(DepDelay > 10) * 1000 AS c3 FROM ontime WHERE Year = 2007 GROUP BY Carrier ORDER BY Carrier - -Q6. Let's try the same query for wide range of years 2000-2008. - -.. code-block:: sql - - SELECT Carrier, c, c2, c*1000/c2 as c3 - FROM - ( - SELECT - Carrier, - count(*) AS c - FROM ontime - WHERE DepDelay>10 - AND Year >= 2000 AND Year <= 2008 - GROUP BY Carrier - ) - ANY INNER JOIN - ( - SELECT - Carrier, - count(*) AS c2 - FROM ontime - WHERE Year >= 2000 AND Year <= 2008 - GROUP BY Carrier - ) USING Carrier - ORDER BY c3 DESC; - -More optimal version of same query: - -.. code-block:: sql - - SELECT Carrier, avg(DepDelay > 10) * 1000 AS c3 FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY Carrier ORDER BY Carrier - -Q7. Percent of delayed (more 10mins) flights per year. - -.. code-block:: sql - - SELECT Year, c1/c2 - FROM - ( - select - Year, - count(*)*1000 as c1 - from ontime - WHERE DepDelay>10 - GROUP BY Year - ) - ANY INNER JOIN - ( - select - Year, - count(*) as c2 - from ontime - GROUP BY Year - ) USING (Year) - ORDER BY Year - -More optimal version of same query: - -.. code-block:: sql - - SELECT Year, avg(DepDelay > 10) FROM ontime GROUP BY Year ORDER BY Year - -Q8. Most popular destination in sense count of direct connected cities for different range of years. - -.. code-block:: sql - - SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime WHERE Year >= 2000 and Year <= 2010 GROUP BY DestCityName ORDER BY u DESC LIMIT 10; - -Q9. - -.. code-block:: sql - - select Year, count(*) as c1 from ontime group by Year; - -Q10. - -.. code-block:: sql - - select - min(Year), max(Year), Carrier, count(*) as cnt, - sum(ArrDelayMinutes>30) as flights_delayed, - round(sum(ArrDelayMinutes>30)/count(*),2) as rate - FROM ontime - WHERE - DayOfWeek not in (6,7) and OriginState not in ('AK', 'HI', 'PR', 'VI') - and DestState not in ('AK', 'HI', 'PR', 'VI') - and FlightDate < '2010-01-01' - GROUP by Carrier - HAVING cnt > 100000 and max(Year) > 1990 - ORDER by rate DESC - LIMIT 1000; - -Bonus: - -.. code-block:: sql - - SELECT avg(cnt) FROM (SELECT Year,Month,count(*) AS cnt FROM ontime WHERE DepDel15=1 GROUP BY Year,Month) - - select avg(c1) from (select Year,Month,count(*) as c1 from ontime group by Year,Month) - - SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime GROUP BY DestCityName ORDER BY u DESC LIMIT 10; - - SELECT OriginCityName, DestCityName, count() AS c FROM ontime GROUP BY OriginCityName, DestCityName ORDER BY c DESC LIMIT 10; - - SELECT OriginCityName, count() AS c FROM ontime GROUP BY OriginCityName ORDER BY c DESC LIMIT 10; diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md new file mode 100644 index 00000000000..8807de3e670 --- /dev/null +++ b/docs/en/getting_started/example_datasets/star_schema.md @@ -0,0 +1,84 @@ +# Star scheme + +Compiling dbgen: + +```bash +git clone git@github.com:vadimtk/ssb-dbgen.git +cd ssb-dbgen +make +``` + +There will be some warnings during the process, but this is normal. + +Place ` dbgen` and ` dists.dss` in any location with 800 GB of free disk space. + +Generating data: + +```bash +./dbgen -s 1000 -T c +./dbgen -s 1000 -T l +``` + +Creating tables in ClickHouse: + +```sql +CREATE TABLE lineorder ( + LO_ORDERKEY UInt32, + LO_LINENUMBER UInt8, + LO_CUSTKEY UInt32, + LO_PARTKEY UInt32, + LO_SUPPKEY UInt32, + LO_ORDERDATE Date, + LO_ORDERPRIORITY String, + LO_SHIPPRIORITY UInt8, + LO_QUANTITY UInt8, + LO_EXTENDEDPRICE UInt32, + LO_ORDTOTALPRICE UInt32, + LO_DISCOUNT UInt8, + LO_REVENUE UInt32, + LO_SUPPLYCOST UInt32, + LO_TAX UInt8, + LO_COMMITDATE Date, + LO_SHIPMODE String +)Engine=MergeTree(LO_ORDERDATE,(LO_ORDERKEY,LO_LINENUMBER,LO_ORDERDATE),8192); + +CREATE TABLE customer ( + C_CUSTKEY UInt32, + C_NAME String, + C_ADDRESS String, + C_CITY String, + C_NATION String, + C_REGION String, + C_PHONE String, + C_MKTSEGMENT String, + C_FAKEDATE Date +)Engine=MergeTree(C_FAKEDATE,(C_CUSTKEY,C_FAKEDATE),8192); + +CREATE TABLE part ( + P_PARTKEY UInt32, + P_NAME String, + P_MFGR String, + P_CATEGORY String, + P_BRAND String, + P_COLOR String, + P_TYPE String, + P_SIZE UInt8, + P_CONTAINER String, + P_FAKEDATE Date +)Engine=MergeTree(P_FAKEDATE,(P_PARTKEY,P_FAKEDATE),8192); + +CREATE TABLE lineorderd AS lineorder ENGINE = Distributed(perftest_3shards_1replicas, default, lineorder, rand()); +CREATE TABLE customerd AS customer ENGINE = Distributed(perftest_3shards_1replicas, default, customer, rand()); +CREATE TABLE partd AS part ENGINE = Distributed(perftest_3shards_1replicas, default, part, rand()); +``` + +For testing on a single server, just use MergeTree tables. +For distributed testing, you need to configure the ` perftest_3shards_1replicas` cluster in the config file. +Next, create MergeTree tables on each server and a Distributed above them. + +Downloading data (change 'customer' to 'customerd' in the distributed version): + +```bash +cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV" +cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" +``` diff --git a/docs/en/getting_started/example_datasets/star_schema.rst b/docs/en/getting_started/example_datasets/star_schema.rst deleted file mode 100644 index 0e041fb343b..00000000000 --- a/docs/en/getting_started/example_datasets/star_schema.rst +++ /dev/null @@ -1,86 +0,0 @@ -Star Schema -=========== - -Compile dbgen: https://github.com/vadimtk/ssb-dbgen - -.. code-block:: bash - - git clone git@github.com:vadimtk/ssb-dbgen.git - cd ssb-dbgen - make - -You will see some warnings. It's Ok. - -Place ``dbgen`` and ``dists.dss`` to some place with at least 800 GB free space available. - -Generate data: - -.. code-block:: bash - - ./dbgen -s 1000 -T c - ./dbgen -s 1000 -T l - -Create tables in ClickHouse: - -.. code-block:: sql - - CREATE TABLE lineorder ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY String, - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE String - ) Engine = MergeTree(LO_ORDERDATE,(LO_ORDERKEY,LO_LINENUMBER,LO_ORDERDATE),8192); - - CREATE TABLE customer ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY String, - C_NATION String, - C_REGION String, - C_PHONE String, - C_MKTSEGMENT String, - C_FAKEDATE Date - ) Engine = MergeTree(C_FAKEDATE,(C_CUSTKEY,C_FAKEDATE),8192); - - CREATE TABLE part ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR String, - P_CATEGORY String, - P_BRAND String, - P_COLOR String, - P_TYPE String, - P_SIZE UInt8, - P_CONTAINER String, - P_FAKEDATE Date - ) Engine = MergeTree(P_FAKEDATE,(P_PARTKEY,P_FAKEDATE),8192); - - CREATE TABLE lineorderd AS lineorder ENGINE = Distributed(perftest_3shards_1replicas, default, lineorder, rand()); - CREATE TABLE customerd AS customer ENGINE = Distributed(perftest_3shards_1replicas, default, customer, rand()); - CREATE TABLE partd AS part ENGINE = Distributed(perftest_3shards_1replicas, default, part, rand()); - -For single-node setup, create just MergeTree tables. -For Distributed setup, you must configure cluster ``perftest_3shards_1replicas`` in configuration file. -Then create MergeTree tables on each node and then create Distributed tables. - -Load data (change customer to customerd in case of distributed setup): - -.. code-block:: bash - - cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV" - cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" - diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md new file mode 100644 index 00000000000..6cbc3b15561 --- /dev/null +++ b/docs/en/getting_started/example_datasets/wikistat.md @@ -0,0 +1,27 @@ +# WikiStat + +See: + +Creating a table: + +```sql +CREATE TABLE wikistat +( + date Date, + time DateTime, + project String, + subproject String, + path String, + hits UInt64, + size UInt64 +) ENGINE = MergeTree(date, (path, time), 8192); +``` + +Loading data: + +```bash +for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sS "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt +cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done +ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done +``` + diff --git a/docs/en/getting_started/example_datasets/wikistat.rst b/docs/en/getting_started/example_datasets/wikistat.rst deleted file mode 100644 index 0e9a8b160c2..00000000000 --- a/docs/en/getting_started/example_datasets/wikistat.rst +++ /dev/null @@ -1,27 +0,0 @@ -WikiStat -======== - -See: https://dumps.wikimedia.org/other/pagecounts-raw/ - -Create table: - -.. code-block:: sql - - CREATE TABLE wikistat - ( - date Date, - time DateTime, - project String, - subproject String, - path String, - hits UInt64, - size UInt64 - ) ENGINE = MergeTree(date, (path, time), 8192); - -Load data: - -.. code-block:: bash - - for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "https://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt - cat links.txt | while read link; do wget https://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done - ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md new file mode 100644 index 00000000000..ded8971d8e2 --- /dev/null +++ b/docs/en/getting_started/index.md @@ -0,0 +1,143 @@ +# Getting started + +## System requirements + +This is not a cross-platform system. It requires Linux Ubuntu Precise (12.04) or newer, with x86_64 architecture and support for the SSE 4.2 instruction set. +To check for SSE 4.2: + +```bash +grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +We recommend using Ubuntu Trusty, Ubuntu Xenial, or Ubuntu Precise. +The terminal must use UTF-8 encoding (the default in Ubuntu). + +## Installation + +For testing and development, the system can be installed on a single server or on a desktop computer. + +### Installing from packages + +In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository: + +```text +deb http://repo.yandex.ru/clickhouse/trusty stable main +``` + +On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`. +If you want to use the most recent test version, replace 'stable' with 'testing'. + +Then run: + +```bash +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server-common +``` + +You can also download and install packages manually from here: + +ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml'). +By default, access is allowed from anywhere for the 'default' user, without a password. See 'user/default/networks'. +For more information, see the section "Configuration files". + +### Installing from sources + +To compile, follow the instructions: build.md + +You can compile packages and install them. +You can also use programs without installing packages. + +```text +Клиент: dbms/src/Client/ +Сервер: dbms/src/Server/ +``` + +For the server, create a catalog with data, such as: + +```text +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +(Configurable in the server config.) +Run 'chown' for the desired user. + +Note the path to logs in the server config (src/dbms/src/Server/config.xml). + +### Other installation methods + +Docker image: + +RPM packages for CentOS or RHEL: + +Gentoo overlay: + +## Launch + +To start the server (as a daemon), run: + +```bash +sudo service clickhouse-server start +``` + +See the logs in the `/var/log/clickhouse-server/ directory.` + +If the server doesn't start, check the configurations in the file `/etc/clickhouse-server/config.xml.` + +You can also launch the server from the console: + +```bash +clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +In this case, the log will be printed to the console, which is convenient during development. +If the configuration file is in the current directory, you don't need to specify the '--config-file' parameter. By default, it uses './config.xml'. + +You can use the command-line client to connect to the server: + +```bash +clickhouse-client +``` + +The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password. +The client can be used for connecting to a remote server. Example: + +```bash +clickhouse-client --host=example.com +``` + +For more information, see the section "Command-line client". + +Checking the system: + +```bash +milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**Congratulations, the system works!** + +To continue experimenting, you can try to download from the test data sets: + +```eval_rst +.. toctree:: + :glob: + + example_datasets/* +``` + diff --git a/docs/en/getting_started/index.rst b/docs/en/getting_started/index.rst deleted file mode 100644 index af8946f76b4..00000000000 --- a/docs/en/getting_started/index.rst +++ /dev/null @@ -1,147 +0,0 @@ -Getting started -=============== - -System requirements -------------------- - -This is not a cross-platform system. It requires Linux Ubuntu Precise (12.04) or newer, x86_64 architecture with SSE 4.2 instruction set. -To test for SSE 4.2 support, do: - -.. code-block:: text - - grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" - -We recommend using Ubuntu Trusty or Ubuntu Xenial or Ubuntu Precise. -The terminal must use UTF-8 encoding (the default in Ubuntu). - -Installation ------------- - -For testing and development, the system can be installed on a single server or on a desktop computer. - -Installing from packages -~~~~~~~~~~~~~~~~~~~~~~~~ - -In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository: - -.. code-block:: text - - deb http://repo.yandex.ru/clickhouse/trusty stable main - -For other Ubuntu versions, replace `trusty` to `xenial` or `precise`. -If you want to use the most fresh testing version of ClickHouse, replace `stable` to `testing`. - -Then run: - -.. code-block:: bash - - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 # optional - sudo apt-get update - sudo apt-get install clickhouse-client clickhouse-server-common - -You can also download and install packages manually from here: -http://repo.yandex.ru/clickhouse/trusty/pool/main/c/clickhouse/, -http://repo.yandex.ru/clickhouse/xenial/pool/main/c/clickhouse/, -http://repo.yandex.ru/clickhouse/precise/pool/main/c/clickhouse/. - -ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml'). -By default, access is allowed from everywhere for the default user without a password. See 'user/default/networks'. For more information, see the section "Configuration files". - -Installing from source -~~~~~~~~~~~~~~~~~~~~~~ -To build, follow the instructions in build.md (for Linux) or in build_osx.md (for Mac OS X). - -You can compile packages and install them. You can also use programs without installing packages. - -.. code-block:: text - - Client: dbms/src/Client/ - Server: dbms/src/Server/ - -For the server, create a directory with data, such as: - -.. code-block:: text - - /opt/clickhouse/data/default/ - /opt/clickhouse/metadata/default/ - -(Configured in the server config.) -Run 'chown' for the desired user. - -Note the path to logs in the server config (src/dbms/src/Server/config.xml). - -Other methods of installation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The Docker image is located here: https://hub.docker.com/r/yandex/clickhouse-server/ - -There are RPM packages for CentOS, RHEL: https://github.com/Altinity/clickhouse-rpm-install - -There is Gentoo overlay located here: https://github.com/kmeaw/clickhouse-overlay - - -Launch ------- - -To start the server (as a daemon), run: - -.. code-block:: bash - - sudo service clickhouse-server start - -View the logs in the directory `/var/log/clickhouse-server/` - -If the server doesn't start, check the configurations in the file `/etc/clickhouse-server/config.xml` - -You can also launch the server from the console: - -.. code-block:: bash - - clickhouse-server --config-file=/etc/clickhouse-server/config.xml - -In this case, the log will be printed to the console, which is convenient during development. If the configuration file is in the current directory, you don't need to specify the '--config-file' parameter. By default, it uses './config.xml'. - -You can use the command-line client to connect to the server: - -.. code-block:: bash - - clickhouse-client - -The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password. -The client can be used for connecting to a remote server. For example: - -.. code-block:: bash - - clickhouse-client --host=example.com - -For more information, see the section "Command-line client". - -Checking the system: - -.. code-block:: bash - - milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client - ClickHouse client version 0.0.18749. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.18749. - - :) SELECT 1 - - SELECT 1 - - ┌─1─┐ - │ 1 │ - └───┘ - - 1 rows in set. Elapsed: 0.003 sec. - - :) - -**Congratulations, it works!** - -For further experiments you can try playing with one of the following example datasets: - -.. toctree:: - :glob: - - example_datasets/* - diff --git a/docs/en/index.rst b/docs/en/index.md similarity index 91% rename from docs/en/index.rst rename to docs/en/index.md index 0e02da517d8..7a550b3d1fd 100644 --- a/docs/en/index.rst +++ b/docs/en/index.md @@ -1,6 +1,6 @@ -Documentation -------------- +# Documentation +```eval_rst .. toctree:: :maxdepth: 6 @@ -20,3 +20,5 @@ Documentation operations/index development/index roadmap +``` + diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md new file mode 100644 index 00000000000..4fd998fed66 --- /dev/null +++ b/docs/en/interfaces/cli.md @@ -0,0 +1,112 @@ +# Command-line client + +To work from the command line, you can use ` clickhouse-client`: + +```bash +$ clickhouse-client +ClickHouse client version 0.0.26176. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.26176.:) +``` + +The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)". + +## Usage + +The client can be used in interactive and non-interactive (batch) mode. +To use batch mode, specify the 'query' parameter, or send data to 'stdin' (it verifies that 'stdin' is not a terminal), or both. +Similar to the HTTP interface, when using the 'query' parameter and sending data to 'stdin', the request is a concatenation of the 'query' parameter, a line feed, and the data in 'stdin'. This is convenient for large INSERT queries. + +Example of using the client to insert data: + +```bash +echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; + +cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; +3, 'some text', '2016-08-14 00:00:00' +4, 'some more text', '2016-08-14 00:00:01' +_EOF + +cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; +``` +In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query. + +By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the --multiquery parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators. +Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program. + +In interactive mode, you get a command line where you can enter queries. + +If 'multiline' is not specified (the default):To run the query, press Enter. The semicolon is not necessary at the end of the query. To enter a multiline query, enter a backslash `\` before the line feed. After you press Enter, you will be asked to enter the next line of the query. + +If multiline is specified:To run a query, end it with a semicolon and press Enter. If the semicolon was omitted at the end of the entered line, you will be asked to enter the next line of the query. + +Only a single query is run, so everything after the semicolon is ignored. + +You can specify `\G` instead of or after the semicolon. This indicates Vertical format. In this format, each value is printed on a separate line, which is convenient for wide tables. This unusual feature was added for compatibility with the MySQL CLI. + +The command line is based on 'readline' (and 'history' or 'libedit', or without a library, depending on the build). In other words, it uses the familiar keyboard shortcuts and keeps a history. +The history is written to `~/.clickhouse-client-history`. + +By default, the format used is PrettyCompact. You can change the format in the FORMAT clause of the query, or by specifying `\G` at the end of the query, using the `--format` or `--vertical` argument in the command line, or using the client configuration file. + +To exit the client, press Ctrl+D (or Ctrl+C), or enter one of the following instead of a query:"exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "q", "Q", ":q", "й", "Й", "Жй" + +When processing a query, the client shows: + +1. Progress, which is updated no more than 10 times per second (by default). For quick queries, the progress might not have time to be displayed. +2. The formatted query after parsing, for debugging. +3. The result in the specified format. +4. The number of lines in the result, the time passed, and the average speed of query processing. + +You can cancel a long query by pressing Ctrl+C. However, you will still need to wait a little for the server to abort the request. It is not possible to cancel a query at certain stages. If you don't wait and press Ctrl+C a second time, the client will exit. + +The command-line client allows passing external data (external temporary tables) for querying. For more information, see the section "External data for query processing". + + + +## Configure + +You can pass parameters to `clickhouse-client` (all parameters have a default value) using: + +- From the Command Line + + Command-line options override the default values and settings in configuration files. + +- Configuration files. + + Settings in the configuration files override the default values. + +### Command line options + +- `--host, -h` -– The server name, 'localhost' by default. You can use either the name or the IPv4 or IPv6 address. +- `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. +- `--user, -u` – The username. Default value: default. +- `--password` – The password. Default value: empty string. +- `--query, -q` – The query to process when using non-interactive mode. +- `--database, -d` – Select the current default database. Default value: the current database from the server settings ('default' by default). +- `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). +- `--multiquery, -n` – If specified, allow processing multiple queries separated by commas. Only works in non-interactive mode. +- `--format, -f` – Use the specified default format to output the result. +- `--vertical, -E` – If specified, use the Vertical format by default to output the result. This is the same as '--format=Vertical'. In this format, each value is printed on a separate line, which is helpful when displaying wide tables. +- `--time, -t` – If specified, print the query execution time to 'stderr' in non-interactive mode. +- `--stacktrace` – If specified, also print the stack trace if an exception occurs. +- `-config-file` – The name of the configuration file. + +### Configuration files + +`clickhouse-client` uses the first existing file of the following: + +- Defined in the `-config-file` parameter. +- `./clickhouse-client.xml` +- `\~/.clickhouse-client/config.xml` +- `/etc/clickhouse-client/config.xml` + +Example of a config file: + +```xml + + username + password + +``` + diff --git a/docs/en/interfaces/cli.rst b/docs/en/interfaces/cli.rst deleted file mode 100644 index 98c53543fc6..00000000000 --- a/docs/en/interfaces/cli.rst +++ /dev/null @@ -1,110 +0,0 @@ -Command-line client -------------------- -To work for command line you can use ``clickhouse-client``: - -.. code-block:: bash - - $ clickhouse-client - ClickHouse client version 0.0.26176. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.26176. - - :) SELECT 1 - - -The ``clickhouse-client`` program accepts the following parameters, which are all optional: - -``--host, -h`` - server name, by defaul - localhost. -You can use either the name or the IPv4 or IPv6 address. - -``--port`` - The port to connect to, by default - 9000. -Note that the HTTP interface and the native interface use different ports. - -``--user, -u`` - The username, by default - default. - -``--password`` - The password, by default - empty string. - -``--query, -q`` - Query to process when using non-interactive mode. - -``--database, -d`` - Select the current default database, by default - the current DB from the server settings (by default, the 'default' DB). - -``--multiline, -m`` - If specified, allow multiline queries (do not send request on Enter). - -``--multiquery, -n`` - If specified, allow processing multiple queries separated by semicolons. -Only works in non-interactive mode. - -``--format, -f`` - Use the specified default format to output the result. -``--vertical, -E`` - If specified, use the Vertical format by default to output the result. This is the same as '--format=Vertical'. In this format, each value is printed on a separate line, which is helpful when displaying wide tables. -``--time, -t`` - If specified, print the query execution time to 'stderr' in non-interactive mode. -``--stacktrace`` - If specified, also prints the stack trace if an exception occurs. -``--config-file`` - Name of the configuration file that has additional settings or changed defaults for the settings listed above. -By default, files are searched for in this order: - -.. code-block:: text - - ./clickhouse-client.xml - ~/.clickhouse-client/config.xml - /etc/clickhouse-client/config.xml - -Settings are only taken from the first file found. - -Example of config file: - -.. code-block:: xml - - - username - password - - -You can also specify any settings that will be used for processing queries. For example, ``clickhouse-client --max_threads=1``. For more information, see the section "Settings". - -The client can be used in interactive and non-interactive (batch) mode. -To use batch mode, specify the 'query' parameter, or send data to 'stdin' (it verifies that 'stdin' is not a terminal), or both. -Similar to the HTTP interface, when using the 'query' parameter and sending data to 'stdin', the request is a concatenation of the 'query' parameter, a line break, and the data in 'stdin'. This is convenient for large INSERT queries. - -Examples for insert data via clickhouse-client: - -.. code-block:: bash - - echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; - - cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; - 3, 'some text', '2016-08-14 00:00:00' - 4, 'some more text', '2016-08-14 00:00:01' - _EOF - - cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; - - -In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query. - -By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the 'multiquery' parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators. -Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program. - -In interactive mode, you get a command line where you can enter queries. - -If 'multiline' is not specified (the default): -To run a query, press Enter. The semicolon is not necessary at the end of the query. To enter a multiline query, enter a backslash ``\`` before the line break - after you press Enter, you will be asked to enter the next line of the query. - -If 'multiline' is specified: -To run a query, end it with a semicolon and press Enter. If the semicolon was omitted at the end of the entered line, you will be asked to enter the next line of the query. - -You can specify ``\G`` instead of or after the semicolon. This indicates using Vertical format. In this format, each value is printed on a separate line, which is convenient for wide tables. This unusual feature was added for compatibility with the MySQL CLI. - -The command line is based on 'readline' (and 'history') (or 'libedit', or even nothing, depending on build). In other words, it uses the familiar keyboard shortcuts and keeps a history. The history is written to /.clickhouse-client-history. - -By default, the format used is PrettyCompact. You can change the format in the FORMAT clause of the query, or by specifying '\G' at the end of the query, using the '--format' or '--vertical' argument in the command line, or using the client configuration file. - -To exit the client, press Ctrl+D (or Ctrl+C), or enter one of the following : -"exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "\q", "\Q", ":q", "\й", "\Й", "Жй" - -When processing a query, the client shows: -#. Progress, which is updated no more than 10 times per second (by default). For quick queries, the progress might not have time to be displayed. -#. The formatted query after parsing, for debugging. -#. The result in the specified format. -#. The number of lines in the result, the time passed, and the average speed of query processing. - -To cancel a lengthy query, press Ctrl+C. However, you will still need to wait a little for the server to abort the request. It is not possible to cancel a query at certain stages. If you don't wait and press Ctrl+C a second time, the client will exit. - -The command-line client allows passing external data (external temporary tables) for querying. For more information, see the section "External data for request processing". diff --git a/docs/en/interfaces/http_interface.rst b/docs/en/interfaces/http_interface.md similarity index 50% rename from docs/en/interfaces/http_interface.rst rename to docs/en/interfaces/http_interface.md index b67e97bb4e7..91c6790f975 100644 --- a/docs/en/interfaces/http_interface.rst +++ b/docs/en/interfaces/http_interface.md @@ -1,17 +1,16 @@ -HTTP interface -============== +# HTTP interface The HTTP interface lets you use ClickHouse on any platform from any programming language. We use it for working from Java and Perl, as well as shell scripts. In other departments, the HTTP interface is used from Perl, Python, and Go. The HTTP interface is more limited than the native interface, but it has better compatibility. By default, clickhouse-server listens for HTTP on port 8123 (this can be changed in the config). -If you make a GET / request without parameters, it returns the string "Ok" (with a line break at the end). You can use this in health-check scripts. +If you make a GET / request without parameters, it returns the string "Ok" (with a line feed at the end). You can use this in health-check scripts. -.. code-block:: bash +```bash +$ curl 'http://localhost:8123/' +Ok. +``` - $ curl 'http://localhost:8123/' - Ok. - -Send the request as a URL 'query' parameter, or as a POST. Or send the beginning of the request in the 'query' parameter, and the rest in the POST (we'll explain later why this is necessary). URL length is limited by 16KB, this limit should be taken into account when sending long queries in the 'query' parameter. +Send the request as a URL 'query' parameter, or as a POST. Or send the beginning of the query in the 'query' parameter, and the rest in the POST (we'll explain later why this is necessary). The size of the URL is limited to 16 KB, so keep this in mind when sending large queries. If successful, you receive the 200 response code and the result in the response body. If an error occurs, you receive the 500 response code and an error description text in the response body. @@ -20,120 +19,118 @@ When using the GET method, 'readonly' is set. In other words, for queries that m Examples: -.. code-block:: bash +```bash +$ curl 'http://localhost:8123/?query=SELECT%201' +1 - $ curl 'http://localhost:8123/?query=SELECT%201' - 1 +$ wget -O- -q 'http://localhost:8123/?query=SELECT 1' +1 - $ wget -O- -q 'http://localhost:8123/?query=SELECT 1' - 1 +$ GET 'http://localhost:8123/?query=SELECT 1' +1 - $ GET 'http://localhost:8123/?query=SELECT 1' - 1 +$ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 +HTTP/1.0 200 OK +Connection: Close +Date: Fri, 16 Nov 2012 19:21:50 GMT - $ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 - HTTP/1.0 200 OK - Connection: Close - Date: Fri, 16 Nov 2012 19:21:50 GMT +1 +``` - 1 +As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. -As you can see, curl is not very convenient because spaces have to be URL-escaped. Although wget escapes everything on its own, we don't recommend it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. +```bash +$ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- +1 -.. code-block:: bash +$ echo 'SELECT 1' | curl 'http://localhost:8123/?query=' --data-binary @- +1 - $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- - 1 +$ echo '1' | curl 'http://localhost:8123/?query=SELECT' --data-binary @- +1 +``` - $ echo 'SELECT 1' | curl 'http://localhost:8123/?query=' --data-binary @- - 1 - - $ echo '1' | curl 'http://localhost:8123/?query=SELECT' --data-binary @- - 1 - -If part of the query is sent in the parameter, and part in the POST, a line break is inserted between these two data parts. +If part of the query is sent in the parameter, and part in the POST, a line feed is inserted between these two data parts. Example (this won't work): -.. code-block:: bash - - $ echo 'ECT 1' | curl 'http://localhost:8123/?query=SEL' --data-binary @- - Code: 59, e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL - ECT 1 - , expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception +```bash +$ echo 'ECT 1' | curl 'http://localhost:8123/?query=SEL' --data-binary @- +Code: 59, e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL +ECT 1 +, expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception +``` By default, data is returned in TabSeparated format (for more information, see the "Formats" section). You use the FORMAT clause of the query to request any other format. -.. code-block:: bash - - $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- - ┏━━━┓ - ┃ 1 ┃ - ┡━━━┩ - │ 1 │ - └───┘ +```bash +$ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- +┏━━━┓ +┃ 1 ┃ +┡━━━┩ +│ 1 │ +└───┘ +``` The POST method of transmitting data is necessary for INSERT queries. In this case, you can write the beginning of the query in the URL parameter, and use POST to pass the data to insert. The data to insert could be, for example, a tab-separated dump from MySQL. In this way, the INSERT query replaces LOAD DATA LOCAL INFILE from MySQL. -Examples: +Examples: Creating a table: -Creating a table: - -.. code-block:: bash - - echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | POST 'http://localhost:8123/' +```bash +echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | POST 'http://localhost:8123/' +``` Using the familiar INSERT query for data insertion: -.. code-block:: bash - - echo 'INSERT INTO t VALUES (1),(2),(3)' | POST 'http://localhost:8123/' +```bash +echo 'INSERT INTO t VALUES (1),(2),(3)' | POST 'http://localhost:8123/' +``` Data can be sent separately from the query: -.. code-block:: bash - - echo '(4),(5),(6)' | POST 'http://localhost:8123/?query=INSERT INTO t VALUES' +```bash +echo '(4),(5),(6)' | POST 'http://localhost:8123/?query=INSERT INTO t VALUES' +``` You can specify any data format. The 'Values' format is the same as what is used when writing INSERT INTO t VALUES: -.. code-block:: bash - - echo '(7),(8),(9)' | POST 'http://localhost:8123/?query=INSERT INTO t FORMAT Values' +```bash +echo '(7),(8),(9)' | POST 'http://localhost:8123/?query=INSERT INTO t FORMAT Values' +``` To insert data from a tab-separated dump, specify the corresponding format: -.. code-block:: bash - - echo -ne '10\n11\n12\n' | POST 'http://localhost:8123/?query=INSERT INTO t FORMAT TabSeparated' +```bash +echo -ne '10\n11\n12\n' | POST 'http://localhost:8123/?query=INSERT INTO t FORMAT TabSeparated' +``` Reading the table contents. Data is output in random order due to parallel query processing: -.. code-block:: bash - - $ GET 'http://localhost:8123/?query=SELECT a FROM t' - 7 - 8 - 9 - 10 - 11 - 12 - 1 - 2 - 3 - 4 - 5 - 6 +```bash +$ GET 'http://localhost:8123/?query=SELECT a FROM t' +7 +8 +9 +10 +11 +12 +1 +2 +3 +4 +5 +6 +``` Deleting the table. -.. code-block:: bash - - POST 'http://localhost:8123/?query=DROP TABLE t' +```bash +POST 'http://localhost:8123/?query=DROP TABLE t' +``` For successful requests that don't return a data table, an empty response body is returned. -You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use a special compressor program to work with it (`sudo apt-get install compressor-metrika-yandex`). +You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex). If you specified 'compress=1' in the URL, the server will compress the data it sends you. If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method. @@ -142,19 +139,19 @@ You can use this to reduce network traffic when transmitting a large amount of d You can use the 'database' URL parameter to specify the default database. -.. code-block:: bash - - $ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- - 0 - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 +```bash +$ echo 'SELECT number FROM numbers LIMIT 10' | curl 'http://localhost:8123/?database=system' --data-binary @- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +``` By default, the database that is registered in the server settings is used as the default database. By default, this is the database called 'default'. Alternatively, you can always specify the database using a dot before the table name. @@ -162,49 +159,57 @@ The username and password can be indicated in one of two ways: 1. Using HTTP Basic Authentication. Example: -.. code-block:: bash - - echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- +```bash +echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- +``` 2. In the 'user' and 'password' URL parameters. Example: -.. code-block:: bash - - echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- - -3. Using 'X-ClickHouse-User' and 'X-ClickHouse-Key' headers. Example: - -.. code-block:: bash - - echo 'SELECT 1' | curl -H "X-ClickHouse-User: user" -H "X-ClickHouse-Key: password" 'http://localhost:8123/' -d @- - +```bash +echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- +``` If the user name is not indicated, the username 'default' is used. If the password is not indicated, an empty password is used. -You can also use the URL parameters to specify any settings for processing a single query, or entire profiles of settings. Example: -`http://localhost:8123/?profile=web&max_rows_to_read=1000000000&query=SELECT+1` +You can also use the URL parameters to specify any settings for processing a single query, or entire profiles of settings. Example:http://localhost:8123/?profile=web&max_rows_to_read=1000000000&query=SELECT+1 For more information, see the section "Settings". -.. code-block:: bash - - $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- - 0 - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 +```bash +$ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:8123/?' --data-binary @- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +``` For information about other parameters, see the section "SET". In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases), and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective. - The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query". -The optional 'quota_key' parameter can be passed as the quota key (any string). It can also be passed as 'X-ClickHouse-Quota' header. For more information, see the section "Quotas". +The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas". The HTTP interface allows passing external data (external temporary tables) for querying. For more information, see the section "External data for query processing". + +## Response buffering + +You can enable response buffering on the server side. The `buffer_size` and `wait_end_of_query` URL parameters are provided for this purpose. + +`buffer_size` determines the number of bytes in the result to buffer in the server memory. If the result body is larger than this threshold, the buffer is written to the HTTP channel, and the remaining data is sent directly to the HTTP channel. + +To ensure that the entire response is buffered, set `wait_end_of_query=1`. In this case, the data that is not stored in memory will be buffered in a temporary server file. + +Example: + +```bash +curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' +``` + +Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage. + diff --git a/docs/en/interfaces/index.rst b/docs/en/interfaces/index.md similarity index 73% rename from docs/en/interfaces/index.rst rename to docs/en/interfaces/index.md index e681fc3dffb..12f2f1ee50f 100644 --- a/docs/en/interfaces/index.rst +++ b/docs/en/interfaces/index.md @@ -1,9 +1,13 @@ -Interfaces -========== + + +# Interfaces To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program. +```eval_rst .. toctree:: :glob: * +``` + diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md new file mode 100644 index 00000000000..08b6c6b055d --- /dev/null +++ b/docs/en/interfaces/jdbc.md @@ -0,0 +1,4 @@ +# JDBC driver + +There is an official JDBC driver for ClickHouse. See [here](https://github.com/yandex/clickhouse-jdbc) . + diff --git a/docs/en/interfaces/jdbc.rst b/docs/en/interfaces/jdbc.rst deleted file mode 100644 index af3c4d1675b..00000000000 --- a/docs/en/interfaces/jdbc.rst +++ /dev/null @@ -1,4 +0,0 @@ -JDBC driver ------------ - -There is official JDBC driver for ClickHouse. See `here `_ . diff --git a/docs/en/interfaces/tcp.rst b/docs/en/interfaces/tcp.md similarity index 82% rename from docs/en/interfaces/tcp.rst rename to docs/en/interfaces/tcp.md index be87c68d927..5b054c3c21c 100644 --- a/docs/en/interfaces/tcp.rst +++ b/docs/en/interfaces/tcp.md @@ -1,4 +1,4 @@ -Native interface (TCP) ----------------------- +# Native interface (TCP) The native interface is used in the "clickhouse-client" command-line client for interaction between servers with distributed query processing, and also in C++ programs. We will only cover the command-line client. + diff --git a/docs/en/interfaces/third-party_client_libraries.md b/docs/en/interfaces/third-party_client_libraries.md new file mode 100644 index 00000000000..cc8ff1f4307 --- /dev/null +++ b/docs/en/interfaces/third-party_client_libraries.md @@ -0,0 +1,38 @@ +# Libraries from third-party developers + +There are libraries for working with ClickHouse for: + +- Python: + - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) + - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) + - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) + - [clickhouse-client](https://github.com/yurial/clickhouse-client) +- PHP + - [clickhouse-php-client](https://github.com/8bitov/clickhouse-php-client) + - [PhpClickHouseClient](https://github.com/SevaCode/PhpClickHouseClient) + - [phpClickHouse](https://github.com/smi2/phpClickHouse) + - [clickhouse-client](https://github.com/bozerkins/clickhouse-client) +- Go + - [clickhouse](https://github.com/kshvakov/clickhouse/) + - [go-clickhouse](https://github.com/roistat/go-clickhouse) + - [mailrugo-clickhouse](https://github.com/mailru/go-clickhouse) + - [golang-clickhouse](https://github.com/leprosus/golang-clickhouse) +- NodeJs + - [clickhouse (NodeJs)](https://github.com/TimonKK/clickhouse) + - [node-clickhouse](https://github.com/apla/node-clickhouse) +- Perl + - [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse) + - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) + - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) +- Ruby + - [clickhouse (Ruby)](https://github.com/archan937/clickhouse) +- R + - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) + - [RClickhouse](https://github.com/IMSMWU/RClickhouse) +- .NET + - [ClickHouse-Net](https://github.com/killwort/ClickHouse-Net) +- C++ + - [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp/) + +We have not tested these libraries. They are listed in random order. + diff --git a/docs/en/interfaces/third-party_client_libraries.rst b/docs/en/interfaces/third-party_client_libraries.rst deleted file mode 100644 index 28ad3ef3beb..00000000000 --- a/docs/en/interfaces/third-party_client_libraries.rst +++ /dev/null @@ -1,37 +0,0 @@ -Third-party client libraries ----------------------------- - -There exist third-party client libraries for ClickHouse: - -* Python: - - `infi.clickhouse_orm `_ - - `sqlalchemy-clickhouse `_ - - `clickhouse-driver `_ - - `clickhouse-client `_ -* PHP - - `clickhouse-php-client `_ - - `PhpClickHouseClient `_ - - `phpClickHouse `_ - - `clickhouse-client `_ -* Go - - `clickhouse `_ - - `go-clickhouse `_ - - `mailru\go-clickhouse `_ - - `golang-clickhouse `_ - - `ch-insert `_ + `ch-encode `_ for data insertion -* NodeJs - - `clickhouse (NodeJs) `_ - - `node-clickhouse `_ -* Perl - - `perl-DBD-ClickHouse `_ - - `HTTP-ClickHouse `_ - - `AnyEvent-ClickHouse `_ -* Ruby - - `clickhouse (Ruby) `_ -* R - - `clickhouse-r `_ - - `RClickhouse `_ -* .NET - - `ClickHouse-Net `_ - -Libraries was not tested by us. Ordering is arbitrary. diff --git a/docs/en/interfaces/third-party_gui.md b/docs/en/interfaces/third-party_gui.md new file mode 100644 index 00000000000..ed471949e5e --- /dev/null +++ b/docs/en/interfaces/third-party_gui.md @@ -0,0 +1,16 @@ +# Visual interfaces from third-party developers + +## Tabix + +Web interface for ClickHouse in the [Tabix](https://github.com/smi2/tabix.ui) project. + +Main features: + +- Works with ClickHouse directly from the browser, without the need to install additional software. +- Query editor with syntax highlighting. +- Auto-completion of commands. +- Tools for graphical analysis of query execution. +- Color scheme options. + +[Tabix documentation](https://tabix.io/doc/). + diff --git a/docs/en/interfaces/third-party_gui.rst b/docs/en/interfaces/third-party_gui.rst deleted file mode 100644 index 05e3a8a90f8..00000000000 --- a/docs/en/interfaces/third-party_gui.rst +++ /dev/null @@ -1,15 +0,0 @@ -Third-party GUI ---------------- - -There are `open source project Tabix `_ company of SMI2, which implements a graphical web interface for ClickHouse. - -Tabix key features: -- works with ClickHouse from the browser directly, without installing additional software; -- query editor that supports highlighting of SQL syntax ClickHouse, auto-completion for all objects, including dictionaries and context-sensitive help for built-in functions. -- graphs, charts and geo-referenced for mapping query results; -- interactive designer PivotTables (pivot) for query results; -- graphical tools for analysis ClickHouse; -- two color theme: light and dark. - - -`Tabix documentation `_ diff --git a/docs/en/introduction/distinctive_features.rst b/docs/en/introduction/distinctive_features.md similarity index 58% rename from docs/en/introduction/distinctive_features.rst rename to docs/en/introduction/distinctive_features.md index f969935b11f..5af3f2d1697 100644 --- a/docs/en/introduction/distinctive_features.rst +++ b/docs/en/introduction/distinctive_features.md @@ -1,62 +1,62 @@ -Distinctive features of ClickHouse -================================== +# Distinctive features of ClickHouse + +## True column-oriented DBMS. -True column-oriented DBMS -------------------------- In a true column-oriented DBMS, there isn't any "garbage" stored with the values. For example, constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data. This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Example are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second. Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows creating tables and databases in runtime, loading data, and running queries without reconfiguring and restarting the server. -Data compression ----------------- +## Data compression + Some column-oriented DBMSs (InfiniDB CE and MonetDB) do not use data compression. However, data compression really improves performance. -Disk storage of data --------------------- -Many column-oriented DBMSs (SAP HANA, and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica. +## Disk storage of data. + +Many column-oriented DBMSs (such as SAP HANA and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica. + +## Parallel processing on multiple cores. -Parallel processing on multiple cores -------------------------------------- Large queries are parallelized in a natural way. -Distributed processing on multiple servers ------------------------------------------- +## Distributed processing on multiple servers. + Almost none of the columnar DBMSs listed above have support for distributed processing. In ClickHouse, data can reside on different shards. Each shard can be a group of replicas that are used for fault tolerance. The query is processed on all the shards in parallel. This is transparent for the user. -SQL support ------------ +## SQL support + If you are familiar with standard SQL, we can't really talk about SQL support. -NULLs are not supported. All the functions have different names. However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances. -JOINs are supported. Subqueries are supported in FROM, IN, JOIN clauses; and scalar subqueries. -Correlated subqueries are not supported. +NULL-s are not supported. All the functions have different names. +However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances. +Support for JOINs. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries. +Dependent subqueries are not supported. -Vector engine -------------- -Data is not only stored by columns, but is processed by vectors - parts of columns. This allows us to achieve high CPU performance. +## Vector engine + +Data is not only stored by columns, but is processed by vectors – parts of columns. This allows us to achieve high CPU performance. + +## Real-time data updates -Real time data updates ----------------------- ClickHouse supports primary key tables. In order to quickly perform queries on the range of the primary key, the data is sorted incrementally using the merge tree. Due to this, data can continually be added to the table. There is no locking when adding data. -Indexes -------- +## Indexes + Having a primary key allows, for example, extracting data for specific clients (Metrica counters) for a specific time range, with low latency less than several dozen milliseconds. -Suitable for online queries ---------------------------- -This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading (in online mode). +## Suitable for online queries -Support for approximated calculations -------------------------------------- +This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading. In other words, in online mode. -#. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles. -#. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk. -#. Supports running an aggregation for a limited number of random keys, instead of for all keys. Under certain conditions for key distribution in the data, this provides a reasonably accurate result while using fewer resources. +## Support for approximated calculations. + +1. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles. +2. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk. +3. Supports running an aggregation for a limited number of random keys, instead of for all keys. Under certain conditions for key distribution in the data, this provides a reasonably accurate result while using fewer resources. + +## Data replication and support for data integrity on replicas + +Uses asynchronous multimaster replication. After being written to any available replica, data is distributed to all the remaining replicas. The system maintains identical data on different replicas. Data is restored automatically after a failure, or using a "button" for complex cases. +For more information, see the section [Data replication](../table_engines/replication.md#table_engines-replication). -Data replication and support for data integrity on replicas ------------------------------------------------------------ -Uses asynchronous multi-master replication. After being written to any available replica, data is distributed to all the remaining replicas. The system maintains identical data on different replicas. Data is restored automatically after a failure, or using a "button" for complex cases. -For more information, see the section "Data replication". diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md new file mode 100644 index 00000000000..80708c02883 --- /dev/null +++ b/docs/en/introduction/features_considered_disadvantages.md @@ -0,0 +1,6 @@ +# ClickHouse features that can be considered disadvantages + +1. No transactions. +2. For aggregation, query results must fit in the RAM on a single server. However, the volume of source data for a query may be indefinitely large. +3. Lack of full-fledged UPDATE/DELETE implementation. + diff --git a/docs/en/introduction/features_considered_disadvantages.rst b/docs/en/introduction/features_considered_disadvantages.rst deleted file mode 100644 index 5c84dfa7642..00000000000 --- a/docs/en/introduction/features_considered_disadvantages.rst +++ /dev/null @@ -1,6 +0,0 @@ -ClickHouse features that can be considered disadvantages --------------------------------------------------------- - -#. No transactions. -#. For aggregation, query results must fit in the RAM on a single server. However, the volume of source data for a query may be indefinitely large. -#. Lack of full-fledged UPDATE/DELETE implementation. diff --git a/docs/en/introduction/index.rst b/docs/en/introduction/index.md similarity index 81% rename from docs/en/introduction/index.rst rename to docs/en/introduction/index.md index e76ffdc3891..68f765a3dd2 100644 --- a/docs/en/introduction/index.rst +++ b/docs/en/introduction/index.md @@ -1,8 +1,7 @@ -Introduction -============ +# Introduction +```eval_rst .. toctree:: - :glob: what_is_clickhouse distinctive_features @@ -10,3 +9,5 @@ Introduction ya_metrika_task possible_silly_questions performance +``` + diff --git a/docs/en/introduction/performance.rst b/docs/en/introduction/performance.md similarity index 76% rename from docs/en/introduction/performance.rst rename to docs/en/introduction/performance.md index f060ae69244..d8958431dfe 100644 --- a/docs/en/introduction/performance.rst +++ b/docs/en/introduction/performance.md @@ -1,21 +1,22 @@ -Performance -=========== -According to internal testing results, ClickHouse shows the best performance for comparable operating scenarios among systems of its class that were available for testing. This includes the highest throughput for long queries, and the lowest latency on short queries. Testing results are shown on this page. +# Performance + +According to internal testing results, ClickHouse shows the best performance for comparable operating scenarios among systems of its class that were available for testing. This includes the highest throughput for long queries, and the lowest latency on short queries. Testing results are shown on a separate page. + +## Throughput for a single large query -Throughput for a single large query ------------------------------------ Throughput can be measured in rows per second or in megabytes per second. If the data is placed in the page cache, a query that is not too complex is processed on modern hardware at a speed of approximately 2-10 GB/s of uncompressed data on a single server (for the simplest cases, the speed may reach 30 GB/s). If data is not placed in the page cache, the speed depends on the disk subsystem and the data compression rate. For example, if the disk subsystem allows reading data at 400 MB/s, and the data compression rate is 3, the speed will be around 1.2 GB/s. To get the speed in rows per second, divide the speed in bytes per second by the total size of the columns used in the query. For example, if 10 bytes of columns are extracted, the speed will be around 100-200 million rows per second. The processing speed increases almost linearly for distributed processing, but only if the number of rows resulting from aggregation or sorting is not too large. -Latency when processing short queries -------------------------------------- -If a query uses a primary key and does not select too many rows to process (hundreds of thousands), and does not use too many columns, we can expect less than 50 milliseconds of latency (single digits of milliseconds in the best case) if data is placed in the page cache. Otherwise, latency is calculated from the number of seeks. If you use rotating drives, for a system that is not overloaded, the latency is calculated by this formula: seek time (10 ms) * number of columns queried * number of data parts. +## Latency when processing short queries + +If a query uses a primary key and does not select too many rows to process (hundreds of thousands), and does not use too many columns, we can expect less than 50 milliseconds of latency (single digits of milliseconds in the best case) if data is placed in the page cache. Otherwise, latency is calculated from the number of seeks. If you use rotating drives, for a system that is not overloaded, the latency is calculated by this formula: seek time (10 ms) \* number of columns queried \* number of data parts. + +## Throughput when processing a large quantity of short queries -Throughput when processing a large quantity of short queries ------------------------------------------------------------- Under the same conditions, ClickHouse can handle several hundred queries per second on a single server (up to several thousand in the best case). Since this scenario is not typical for analytical DBMSs, we recommend expecting a maximum of 100 queries per second. -Performance on data insertion ------------------------------ -We recommend inserting data in packets of at least 1000 rows, or no more than a single request per second. When inserting to a MergeTree table from a tab-separated dump, the insertion speed will be from 50 to 200 MB/s. If the inserted rows are around 1 Kb in size, the speed will be from 50,000 to 200,000 rows per second. If the rows are small, the performance will be higher in rows per second (on Yandex Banner System data -> 500,000 rows per second, on Graphite data -> 1,000,000 rows per second). To improve performance, you can make multiple INSERT queries in parallel, and performance will increase linearly. +## Performance when inserting data + +We recommend inserting data in packets of at least 1000 rows, or no more than a single request per second. When inserting to a MergeTree table from a tab-separated dump, the insertion speed will be from 50 to 200 MB/s. If the inserted rows are around 1 Kb in size, the speed will be from 50,000 to 200,000 rows per second. If the rows are small, the performance will be higher in rows per second (on Banner System data -`>` 500,000 rows per second; on Graphite data -`>` 1,000,000 rows per second). To improve performance, you can make multiple INSERT queries in parallel, and performance will increase linearly. + diff --git a/docs/en/introduction/possible_silly_questions.md b/docs/en/introduction/possible_silly_questions.md new file mode 100644 index 00000000000..3e147a3eb3d --- /dev/null +++ b/docs/en/introduction/possible_silly_questions.md @@ -0,0 +1,15 @@ +# Everything you were afraid to ask + +## Why not use something like MapReduce? + +We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include YAMR, Hadoop, and YT. + +These systems aren't appropriate for online queries due to their high latency. In other words, they can't be used as the back-end for a web interface. +These types of systems aren't useful for real-time data updates. +Distributed sorting isn't the best way to perform reduce operations if the result of the operation and all the intermediate results (if there are any) are located in the RAM of a single server, which is usually the case for online queries. In such a case, a hash table is the optimal way to perform reduce operations. A common approach to optimizing map-reduce tasks is pre-aggregation (partial reduce) using a hash table in RAM. The user performs this optimization manually. +Distributed sorting is one of the main causes of reduced performance when running simple map-reduce tasks. + +Systems like map-reduce allow executing any code on the cluster. But a declarative query language is better suited to OLAP in order to run experiments quickly. For example, Hadoop has Hive and Pig. Also consider Cloudera Impala, Shark (outdated) for Spark, and Spark SQL, Presto, and Apache Drill. Performance when running such tasks is highly sub-optimal compared to specialized systems, but relatively high latency makes it unrealistic to use these systems as the backend for a web interface. + +YT allows storing groups of columns separately. But YT can't be considered a true column-based system because it doesn't have fixed-length data types (for efficiently storing numbers without extra "garbage"), and also due to its lack of a vector engine. Tasks are performed in YT using custom code in streaming mode, so they cannot be optimized enough (up to hundreds of millions of rows per second per server). "Dynamic table sorting" is under development in YT using MergeTree, strict value typing, and a query language similar to SQL. Dynamically sorted tables are not appropriate for OLAP tasks because the data is stored by row. The YT query language is still under development, so we can't yet rely on this functionality. YT developers are considering using dynamically sorted tables in OLTP and Key-Value scenarios. + diff --git a/docs/en/introduction/possible_silly_questions.rst b/docs/en/introduction/possible_silly_questions.rst deleted file mode 100644 index 14d9ceb4293..00000000000 --- a/docs/en/introduction/possible_silly_questions.rst +++ /dev/null @@ -1,19 +0,0 @@ -Possible silly questions ------------------------- - -Why not to use systems like MapReduce? -"""""""""""""""""""""""""""""""""""""" - -Systems like map-reduce are distributed computing systems, where the reduce phase is performed using distributed sorting. -Regarding this aspect, map-reduce is similar to other systems like YAMR, Hadoop, YT. - -These systems are not suitable for online queries because of latency, So they can't be used in backend-level for web interface. -Systems like this also are not suitable for real-time updates. -Distributed sorting is not optimal solution for reduce operations, if the result of the operation and all intermediate results, shall they exist, fit in operational memory of a single server, as usually happens in case of online analytical queries. -In this case the optimal way to perform reduce operations is by using a hash-table. A common optimization method for map-reduce tasks is combine operation (partial reduce) which uses hash-tables in memory. This optimization is done by the user manually. -Distributed sorting is the main reason for long latencies of simple map-reduce jobs. - -Systems similar to map-reduce enable running any code on the cluster. But for OLAP use-cases declarative query languages are better suited as they allow to carry out investigations faster. For example, for Hadoop there are Hive and Pig. There are others: Cloudera Impala, Shark (deprecated) and Spark SQL for Spark, Presto, Apache Drill. -However, performance of such tasks is highly sub-optimal compared to the performance of specialized systems and relatively high latency does not allow the use of these systems as a backend for the web interface. - -YT allows you to store separate groups of columns. But YT is not a truly columnar storage system, as the system has no fixed length data types (so you can efficiently store a number without "garbage"), and there is no vector engine. Tasks in YT are performed by arbitrary code in streaming mode, so can not be sufficiently optimized (up to hundreds of millions of lines per second per server). In 2014-2016 YT is to develop "dynamic table sorting" functionality using Merge Tree, strongly typed values ​​and SQL-like language support. Dynamically sorted tables are not suited for OLAP tasks, since the data is stored in rows. Query language development in YT is still in incubating phase, which does not allow it to focus on this functionality. YT developers are considering dynamically sorted tables for use in OLTP and Key-Value scenarios. diff --git a/docs/en/introduction/what_is_clickhouse.md b/docs/en/introduction/what_is_clickhouse.md new file mode 100644 index 00000000000..52714b810a9 --- /dev/null +++ b/docs/en/introduction/what_is_clickhouse.md @@ -0,0 +1,123 @@ +# What is ClickHouse? + +ClickHouse is a columnar DBMS for OLAP. + +In a "normal" row-oriented DBMS, data is stored in this order: + +```text +5123456789123456789 1 Евробаскет - Греция - Босния и Герцеговина - example.com 1 2011-09-01 01:03:02 6274717 1294101174 11409 612345678912345678 0 33 6 http://www.example.com/basketball/team/123/match/456789.html http://www.example.com/basketball/team/123/match/987654.html 0 1366 768 32 10 3183 0 0 13 0\0 1 1 0 0 2011142 -1 0 0 01321 613 660 2011-09-01 08:01:17 0 0 0 0 utf-8 1466 0 0 0 5678901234567890123 277789954 0 0 0 0 0 +5234985259563631958 0 Консалтинг, налогообложение, бухгалтерский учет, право 1 2011-09-01 01:03:02 6320881 2111222333 213 6458937489576391093 0 3 2 http://www.example.ru/ 0 800 600 16 10 2 153.1 0 0 10 63 1 1 0 0 2111678 000 0 588 368 240 2011-09-01 01:03:17 4 0 60310 0 windows-1251 1466 0 000 778899001 0 0 0 0 0 +... +``` + +In order words, all the values related to a row are stored next to each other. +Examples of a row-oriented DBMS are MySQL, Postgres, MS SQL Server, and others. + +In a column-oriented DBMS, data is stored like this: + +```text +WatchID: 5385521489354350662 5385521490329509958 5385521489953706054 5385521490476781638 5385521490583269446 5385521490218868806 5385521491437850694 5385521491090174022 5385521490792669254 5385521490420695110 5385521491532181574 5385521491559694406 5385521491459625030 5385521492275175494 5385521492781318214 5385521492710027334 5385521492955615302 5385521493708759110 5385521494506434630 5385521493104611398 +JavaEnable: 1 0 1 0 0 0 1 0 1 1 1 1 1 1 0 1 0 0 1 1 +Title: Yandex Announcements - Investor Relations - Yandex Yandex — Contact us — Moscow Yandex — Mission Ru Yandex — History — History of Yandex Yandex Financial Releases - Investor Relations - Yandex Yandex — Locations Yandex Board of Directors - Corporate Governance - Yandex Yandex — Technologies +GoodEvent: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +EventTime: 2016-05-18 05:19:20 2016-05-18 08:10:20 2016-05-18 07:38:00 2016-05-18 01:13:08 2016-05-18 00:04:06 2016-05-18 04:21:30 2016-05-18 00:34:16 2016-05-18 07:35:49 2016-05-18 11:41:59 2016-05-18 01:13:32 +``` + +These examples only show the order that data is arranged in. +The values from different columns are stored separately, and data from the same column is stored together. + +Examples of column-oriented DBMSs: `Vertica`, `Paraccel (Actian Matrix) (Amazon Redshift)`, `Sybase IQ`, `Exasol`, `Infobright`, `InfiniDB`, `MonetDB (VectorWise) (Actian Vector)`, `LucidDB`, `SAP HANA`, `Google Dremel`, `Google PowerDrill`, `Druid`, `kdb+`, and so on. + +Different orders for storing data are better suited to different scenarios. +The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query – rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. + +The higher the load on the system, the more important it is to customize the system to the scenario, and the more specific this customization becomes. There is no system that is equally well-suited to significantly different scenarios. If a system is adaptable to a wide set of scenarios, under a high load, the system will handle all the scenarios equally poorly, or will work well for just one of the scenarios. + +We'll say that the following is true for the OLAP (online analytical processing) scenario: + +- The vast majority of requests are for read access. +- Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. +- Data is added to the DB but is not modified. +- For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. +- Tables are "wide," meaning they contain a large number of columns. +- Queries are relatively rare (usually hundreds of queries per server or less per second). +- For simple queries, latencies around 50 ms are allowed. +- Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). +- Requires high throughput when processing a single query (up to billions of rows per second per server). +- There are no transactions. +- Low requirements for data consistency. +- There is one large table per query. All tables are small, except for one. +- A query result is significantly smaller than the source data. In other words, data is filtered or aggregated. The result fits in a single server's RAM. + +It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn't make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Elliptics for analytics, you will get very poor performance compared to OLAP databases. + +Columnar-oriented databases are better suited to OLAP scenarios (at least 100 times better in processing speed for most queries), for the following reasons: + +1. For I/O. +2. For an analytical query, only a small number of table columns need to be read. In a column-oriented database, you can read just the data you need. For example, if you need 5 columns out of 100, you can expect a 20-fold reduction in I/O. +3. Since data is read in packets, it is easier to compress. Data in columns is also easier to compress. This further reduces the I/O volume. +4. Due to the reduced I/O, more data fits in the system cache. + +For example, the query "count the number of records for each advertising platform" requires reading one "advertising platform ID" column, which takes up 1 byte uncompressed. If most of the traffic was not from advertising platforms, you can expect at least 10-fold compression of this column. When using a quick compression algorithm, data decompression is possible at a speed of at least several gigabytes of uncompressed data per second. In other words, this query can be processed at a speed of approximately several billion rows per second on a single server. This speed is actually achieved in practice. + +Example: + +```bash +milovidov@hostname:~$ clickhouse-client +ClickHouse client version 0.0.52053. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.52053. + +:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 + +SELECT + CounterID, + count() +FROM hits +GROUP BY CounterID +ORDER BY count() DESC +LIMIT 20 + +┌─CounterID─┬──count()─┐ +│ 114208 │ 56057344 │ +│ 115080 │ 51619590 │ +│ 3228 │ 44658301 │ +│ 38230 │ 42045932 │ +│ 145263 │ 42042158 │ +│ 91244 │ 38297270 │ +│ 154139 │ 26647572 │ +│ 150748 │ 24112755 │ +│ 242232 │ 21302571 │ +│ 338158 │ 13507087 │ +│ 62180 │ 12229491 │ +│ 82264 │ 12187441 │ +│ 232261 │ 12148031 │ +│ 146272 │ 11438516 │ +│ 168777 │ 11403636 │ +│ 4120072 │ 11227824 │ +│ 10938808 │ 10519739 │ +│ 74088 │ 9047015 │ +│ 115079 │ 8837972 │ +│ 337234 │ 8205961 │ +└───────────┴──────────┘ + +20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) + +:) +``` + +2. For CPU. + +Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don't do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. +It makes sense to both store data in columns and process it, when possible, by columns. + +There are two ways to do this: + +1. A vector engine. All operations are written for vectors, instead of for separate values. This means you don't need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. + +2. Code generation. The code generated for the query has all the indirect calls in it. + +This is not done in "normal" databases, because it doesn't make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) + +Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization. + diff --git a/docs/en/introduction/what_is_clickhouse.rst b/docs/en/introduction/what_is_clickhouse.rst deleted file mode 100644 index f9029eb8ccb..00000000000 --- a/docs/en/introduction/what_is_clickhouse.rst +++ /dev/null @@ -1,122 +0,0 @@ -What is ClickHouse? -=================== - -ClickHouse is a columnar DBMS for OLAP. - -In a "normal" row-oriented DBMS, data is stored in this order: - -.. code-block:: text - - 5123456789123456789 1 Eurobasket - Greece - Bosnia and Herzegovina - example.com 1 2011-09-01 01:03:02 6274717 1294101174 11409 612345678912345678 0 33 6 http://www.example.com/basketball/team/123/match/456789.html http://www.example.com/basketball/team/123/match/987654.html 0 1366 768 32 10 3183 0 0 13 0\0 1 1 0 0 2011142 -1 0 0 01321 613 660 2011-09-01 08:01:17 0 0 0 0 utf-8 1466 0 0 0 5678901234567890123 277789954 0 0 0 0 0 - 5234985259563631958 0 Consulting, Tax assessment, Accounting, Law 1 2011-09-01 01:03:02 6320881 2111222333 213 6458937489576391093 0 3 2 http://www.example.ru/ 0 800 600 16 10 2 153.1 0 0 10 63 1 1 0 0 2111678 000 0 588 368 240 2011-09-01 01:03:17 4 0 60310 0 windows-1251 1466 0 000 778899001 0 0 0 0 0 - ... - - -In other words, all the values related to a row are stored next to each other. Examples of a row-oriented DBMS are MySQL, Postgres, MS SQL Server, and others. - -In a column-oriented DBMS, data is stored like this: - -.. code-block:: text - - WatchID: 5385521489354350662 5385521490329509958 5385521489953706054 5385521490476781638 5385521490583269446 5385521490218868806 5385521491437850694 5385521491090174022 5385521490792669254 5385521490420695110 5385521491532181574 5385521491559694406 5385521491459625030 5385521492275175494 5385521492781318214 5385521492710027334 5385521492955615302 5385521493708759110 5385521494506434630 5385521493104611398 - JavaEnable: 1 0 1 0 0 0 1 0 1 1 1 1 1 1 0 1 0 0 1 1 - Title: Yandex Announcements - Investor Relations - Yandex Yandex — Contact us — Moscow Yandex — Mission Ru Yandex — History — History of Yandex Yandex Financial Releases - Investor Relations - Yandex Yandex — Locations Yandex Board of Directors - Corporate Governance - Yandex Yandex — Technologies - GoodEvent: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - EventTime: 2016-05-18 05:19:20 2016-05-18 08:10:20 2016-05-18 07:38:00 2016-05-18 01:13:08 2016-05-18 00:04:06 2016-05-18 04:21:30 2016-05-18 00:34:16 2016-05-18 07:35:49 2016-05-18 11:41:59 2016-05-18 01:13:32 - -These examples only show the order that data is arranged in. -The values from different columns are stored separately, and data from the same column is stored together. -Examples of a column-oriented DBMS: ``Vertica``, ``Paraccel (Actian Matrix) (Amazon Redshift)``, ``Sybase IQ``, ``Exasol``, ``Infobright``, ``InfiniDB``, ``MonetDB (VectorWise) (Actian Vector)``, ``LucidDB``, ``SAP HANA``, ``Google Dremel``, ``Google PowerDrill``, ``Druid``, ``kdb+``, etc. - -Different orders for storing data are better suited to different scenarios. -The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query - rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. - -The higher the load on the system, the more important it is to customize the system to the scenario, and the more specific this customization becomes. There is no system that is equally well-suited to significantly different scenarios. If a system is adaptable to a wide set of scenarios, under a high load, the system will handle all the scenarios equally poorly, or will work well for just one of the scenarios. - -We'll say that the following is true for the OLAP (online analytical processing) scenario: - -* The vast majority of requests are for read access. -* Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. -* Data is added to the DB but is not modified. -* For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. -* Tables are "wide," meaning they contain a large number of columns. -* Queries are relatively rare (usually hundreds of queries per server or less per second). -* For simple queries, latencies around 50 ms are allowed. -* Column values are fairly small - numbers and short strings (for example, 60 bytes per URL). -* Requires high throughput when processing a single query (up to billions of rows per second per server). -* There are no transactions. -* Low requirements for data consistency. -* There is one large table per query. All tables are small, except for one. -* A query result is significantly smaller than the source data. That is, data is filtered or aggregated. The result fits in a single server's RAM. - -It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn't make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Elliptics for analytics, you will get very poor performance compared to OLAP databases. - -Columnar-oriented databases are better suited to OLAP scenarios (at least 100 times better in processing speed for most queries), for the following reasons: - -1. For I/O. - -#. For an analytical query, only a small number of table columns need to be read. In a column-oriented database, you can read just the data you need. For example, if you need 5 columns out of 100, you can expect a 20-fold reduction in I/O. -#. Since data is read in packets, it is easier to compress. Data in columns is also easier to compress. This further reduces the I/O volume. -#. Due to the reduced I/O, more data fits in the system cache. - -For example, the query "count the number of records for each advertising platform" requires reading one "advertising platform ID" column, which takes up 1 byte uncompressed. If most of the traffic was not from advertising platforms, you can expect at least 10-fold compression of this column. When using a quick compression algorithm, data decompression is possible at a speed of at least several gigabytes of uncompressed data per second. In other words, this query can be processed at a speed of approximately several billion rows per second on a single server. This speed is actually achieved in practice. - -Example: - -.. code-block:: text - - milovidov@hostname:~$ clickhouse-client - ClickHouse client version 0.0.52053. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.52053. - - :) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - - SELECT - CounterID, - count() - FROM hits - GROUP BY CounterID - ORDER BY count() DESC - LIMIT 20 - - ┌─CounterID─┬──count()─┐ - │ 114208 │ 56057344 │ - │ 115080 │ 51619590 │ - │ 3228 │ 44658301 │ - │ 38230 │ 42045932 │ - │ 145263 │ 42042158 │ - │ 91244 │ 38297270 │ - │ 154139 │ 26647572 │ - │ 150748 │ 24112755 │ - │ 242232 │ 21302571 │ - │ 338158 │ 13507087 │ - │ 62180 │ 12229491 │ - │ 82264 │ 12187441 │ - │ 232261 │ 12148031 │ - │ 146272 │ 11438516 │ - │ 168777 │ 11403636 │ - │ 4120072 │ 11227824 │ - │ 10938808 │ 10519739 │ - │ 74088 │ 9047015 │ - │ 115079 │ 8837972 │ - │ 337234 │ 8205961 │ - └───────────┴──────────┘ - - 20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - - :) - -2. For CPU. - -Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don't do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. -It makes sense to both store data in columns and process it, when possible, by columns. - -There are two ways to do this: - -#. A vector engine. All operations are written for vectors, instead of for separate values. This means you don't need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. -#. Code generation. The code generated for the query has all the indirect calls in it. - -This is not done in "normal" databases, because it doesn't make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) - -Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization. diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md new file mode 100644 index 00000000000..6a488be9b5f --- /dev/null +++ b/docs/en/introduction/ya_metrika_task.md @@ -0,0 +1,47 @@ +# The Yandex.Metrica task + +ClickHouse currently powers [ Yandex.Metrica](https://metrika.yandex.ru/), [ the second largest platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all) for Web Analytics. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data. + +We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors. +At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds. + +## Usage in Yandex.Metrica and other Yandex services + +ClickHouse is used for multiple purposes in Yandex.Metrica. +Its main task is to build reports in online mode using non-aggregated data. It uses a cluster of 374 servers, which store over 20.3 trillion rows in the database. The volume of compressed data, without counting duplication and replication, is about 2 PB. The volume of uncompressed data (in TSV format) would be approximately 17 PB. + +ClickHouse is also used for: + +- Storing data for Session Replay from Yandex.Metrica. +- Processing intermediate data. +- Building global reports with Analytics. +- Running queries for debugging the Yandex.Metrica engine. +- Analyzing logs from the API and the user interface. + +ClickHouse has at least a dozen installations in other Yandex services: in search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services, and others. + +## Aggregated and non-aggregated data + +There is a popular opinion that in order to effectively calculate statistics, you must aggregate data, since this reduces the volume of data. + +But data aggregation is a very limited solution, for the following reasons: + +- You must have a pre-defined list of reports the user will need. +- The user can't make custom reports. +- When aggregating a large quantity of keys, the volume of data is not reduced, and aggregation is useless. +- For a large number of reports, there are too many aggregation variations (combinatorial explosion). +- When aggregating keys with high cardinality (such as URLs), the volume of data is not reduced by much (less than twofold). +- For this reason, the volume of data with aggregation might grow instead of shrink. +- Users do not view all the reports we generate for them. A large portion of calculations are useless. +- The logical integrity of data may be violated for various aggregations. + +If we do not aggregate anything and work with non-aggregated data, this might actually reduce the volume of calculations. + +However, with aggregation, a significant part of the work is taken offline and completed relatively calmly. In contrast, online calculations require calculating as fast as possible, since the user is waiting for the result. + +Yandex.Metrica has a specialized system for aggregating data called Metrage, which is used for the majority of reports. +Starting in 2009, Yandex.Metrica also used a specialized OLAP database for non-aggregated data called OLAPServer, which was previously used for the report builder. +OLAPServer worked well for non-aggregated data, but it had many restrictions that did not allow it to be used for all reports as desired. These included the lack of support for data types (only numbers), and the inability to incrementally update data in real-time (it could only be done by rewriting data daily). OLAPServer is not a DBMS, but a specialized DB. + +To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS. + diff --git a/docs/en/introduction/ya_metrika_task.rst b/docs/en/introduction/ya_metrika_task.rst deleted file mode 100644 index 5757c844090..00000000000 --- a/docs/en/introduction/ya_metrika_task.rst +++ /dev/null @@ -1,42 +0,0 @@ -The Yandex.Metrica task ------------------------ - -ClickHouse currently powers `Yandex.Metrica `_, world's `second largest `_ web analytics platform, with over 13 trillion database records and over 20 billion events a day, generating customized reports on the fly directly from non-aggregated data. - -We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors. -At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds. - -Usage in Yandex.Metrica and other Yandex services -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -ClickHouse is used for multiple purposes in Yandex.Metrica. Its main task is to build reports in online mode using non-aggregated data. It uses a cluster of 374 servers, which store over 20.3 trillion rows in the database. The volume of compressed data, without counting duplication and replication, is about 2 PB. The volume of uncompressed data (in TSV format) would be approximately 17 PB. - -ClickHouse is also used for: - * Storing WebVisor data. - * Processing intermediate data. - * Building global reports with Analytics. - * Running queries for debugging the Metrica engine. - * Analyzing logs from the API and the user interface. - -ClickHouse has at least a dozen installations in other Yandex services: in search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services, and others. - -Aggregated and non-aggregated data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -There is a popular opinion that in order to effectively calculate statistics, you must aggregate data, since this reduces the volume of data. - -But data aggregation is a very limited solution, for the following reasons: - -* You must have a pre-defined list of reports the user will need. The user can't make custom reports. -* When aggregating a large quantity of keys, the volume of data is not reduced, and aggregation is useless. -* For a large number of reports, there are too many aggregation variations (combinatorial explosion). -* When aggregating keys with high cardinality (such as URLs), the volume of data is not reduced by much (less than twofold). For this reason, the volume of data with aggregation might grow instead of shrink. -* Users will not view all the reports we calculate for them. A large portion of calculations are useless. -* The logical integrity of data may be violated for various aggregations. - -If we do not aggregate anything and work with non-aggregated data, this might actually reduce the volume of calculations. - -However, with aggregation, a significant part of the work is taken offline and completed relatively calmly. In contrast, online calculations require calculating as fast as possible, since the user is waiting for the result. - -Yandex.Metrica has a specialized system for aggregating data called Metrage, which is used for the majority of reports. Starting in 2009, Yandex.Metrica also used a specialized OLAP database for non-aggregated data called OLAPServer, which was previously used for the report builder. OLAPServer worked well for non-aggregated data, but it had many restrictions that did not allow it to be used for all reports as desired. These included the lack of support for data types (only numbers), and the inability to incrementally update data in real-time (it could only be done by rewriting data daily). OLAPServer is not a DBMS, but a specialized DB. - -To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS. diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md new file mode 100644 index 00000000000..e2e79d7f2aa --- /dev/null +++ b/docs/en/operations/access_rights.md @@ -0,0 +1,98 @@ +# Access rights + +Users and access rights are set up in the user config. This is usually `users.xml`. + +Users are recorded in the 'users' section. We'll look at a fragment of the `users.xml` file: + +```xml + + + + + + + + + + + default + + + default + + + + + + + web + default + + test + + +``` + +You can see a declaration from two users: `default`and`web`. We added the `web` user separately. + +The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify `user` and `password` (see the section on the [Distributed](../table_engines/distributed.html) engine). + +The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. + +The password is specified in open format (not recommended) or in SHA-256. The hash isn't salted. In this regard, you should not consider these passwords as providing security against potential malicious attacks. Rather, they are necessary for protection from employees. + +A list of networks is specified that access is allowed from. In this example, the list of networks for both users is loaded from a separate file (/etc/metrika.xml) containing the 'networks' substitution. Here is a fragment of it: + +```xml + + ... + + ::/64 + 203.0.113.0/24 + 2001:DB8::/32 + ... + + +``` + +We could have defined this list of networks directly in 'users.xml', or in a file in the 'users.d' directory (for more information, see the section "Configuration files"). + +The config includes comments explaining how to open access from everywhere. + +For use in production, only specify IP elements (IP addresses and their masks), since using 'host' and 'hoost_regexp' might cause extra latency. + +Next the user settings profile is specified (see the section "Settings profiles"). You can specify the default profile, `default`. The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is 'readonly' set to 1, which provides read-only access. + +After this, the quota is defined (see the section "Quotas"). You can specify the default quota, `default`. It is set in the config by default so that it only counts resource usage, but does not restrict it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually. + +In the optional `` section, you can also specify a list of databases that the user can access. By default, all databases are available to the user. You can specify the `default` database. In this case, the user will receive access to the database by default. + +Access to the `system` database is always allowed (since this database is used for processing queries). + +The user can get a list of all databases and tables in them by using `SHOW` queries or system tables, even if access to individual databases isn't allowed. + +Database access is not related to the [readonly](settings/query_complexity.md#query_complexity_readonly) setting. You can't grant full access to one database and `readonly` access to another one. + diff --git a/docs/en/operations/access_rights.rst b/docs/en/operations/access_rights.rst deleted file mode 100644 index 020bb64f5f5..00000000000 --- a/docs/en/operations/access_rights.rst +++ /dev/null @@ -1,71 +0,0 @@ -Access rights -============= -Users and access rights are set up in the user config. This is usually ``users.xml``. - -Users are recorded in the ``users`` section. Let's look at part of the ``users.xml`` file: - -.. code-block:: xml - - - - - - - - - - - - - default - - default - - - - - - - web - default - - -Here we can see that two users are declared: ``default`` and ``web``. We added the ``web`` user ourselves. -The ``default`` user is chosen in cases when the username is not passed. The ``default`` user can also be used for distributed query processing - the system accesses remote servers using this username if no ``user`` and ``password`` were configured for that server inside cluster configuration (see also section about "Distributed" table engine). - -For connection to the server inside cluster you should use the user without any substantial restrictions or quotas - otherwise, distributed queries will fail. - -The password is specified in plain text directly in the config. In this regard, you should not consider these passwords as providing security against potential malicious attacks. Rather, they are necessary for protection from Yandex employees. - -A list of networks is specified that access is allowed from. In this example, the list of networks for both users is loaded from a separate file (``/etc/metrika.xml``) containing the ``networks`` substitution. Here is a fragment of it: - -.. code-block:: xml - - - ... - - ::/64 - 93.111.222.128/26 - 2a02:6b8:0:111::/64 - ... - - - -We could have defined this list of networks directly in ``users.xml``, or in a file in the ``users.d`` directory (for more information, see the section "Configuration files"). - -The config includes comments explaining how to open access from everywhere. - -For use in production, only specify IP elements (IP addresses and their masks), since using ``host`` and ``host_regexp`` might cause extra latency. - -Next the user settings profile is specified (see the section "Settings profiles"). You can specify the default profile, ``default``. The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is ``readonly`` set to ``1``, which provides read-only access. - -After this, the quota is defined (see the section "Quotas"). You can specify the default quota, ``default``. It is set in the config by default so that it only counts resource usage, but does not restrict it. The quota can have any name. You can specify the same quota for different users - in this case, resource usage is calculated for each user individually. diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md new file mode 100644 index 00000000000..c3122617bf1 --- /dev/null +++ b/docs/en/operations/configuration_files.md @@ -0,0 +1,28 @@ + + +# Configuration files + +The main server config file is `config.xml`. It resides in the `/etc/clickhouse-server/` directory. + +Individual settings can be overridden in the `*.xml`and`*.conf` files in the `conf.d` and `config.d` directories next to the config file. + +The `replace` or `remove` attributes can be specified for the elements of these config files. + +If neither is specified, it combines the contents of elements recursively, replacing values of duplicate children. + +If `replace` is specified, it replaces the entire element with the specified one. + +If ` remove` is specified, it deletes the element. + +The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the config in the [include_from](server_settings/settings.md#server_settings-include_from) element. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional = "true"` attribute (for example, settings for [ macros](server_settings/settings.md#server_settings-macros)). + +Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at ` /path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element. + +The `config.xml` file can specify a separate config with user settings, profiles, and quotas. The relative path to this config is set in the 'users_config' element. By default, it is `users.xml`. If `users_config` is omitted, the user settings, profiles, and quotas are specified directly in `config.xml`. + +In addition, `users_config` may have overrides in files from the `users_config.d` directory (for example, `users.d`) and substitutions. + +For each config file, the server also generates `file-preprocessed.xml` files when starting. These files contain all the completed substitutions and overrides, and they are intended for informational use. If ZooKeeper substitutions were used in the config files but ZooKeeper is not available on the server start, the server loads the configuration from the preprocessed file. + +The server tracks changes in config files, as well as files and ZooKeeper nodes that were used when performing substitutions and overrides, and reloads the settings for users and clusters on the fly. This means that you can modify the cluster, users, and their settings without restarting the server. + diff --git a/docs/en/operations/configuration_files.rst b/docs/en/operations/configuration_files.rst deleted file mode 100644 index e7cea6e52d6..00000000000 --- a/docs/en/operations/configuration_files.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _configuration_files: - -Configuration files -=================== - -The main server config file is ``config.xml``. It resides in the ``/etc/clickhouse-server/`` directory. - -Certain settings can be overridden in the ``*.xml`` and ``*.conf`` files from the ``conf.d`` and ``config.d`` directories next to the config. - -The ``replace`` and ``remove`` attributes can be specified for the elements of these config files. -If neither is specified, it combines the contents of elements recursively, replacing values of duplicate children. -If ``replace`` is specified, it replaces the entire element with the specified one. -If ``remove`` is specified, it deletes the element. - -The config can also define "substitutions". If an element has the ``incl`` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is ``/etc/metrika.xml``. This can be changed in the config in the ``include_from`` element. The substitution values are specified in ``/yandex/substitution_name`` elements of this file. - -You can also perform substitutions from ZooKeeper nodes. To do that add the ``from_zk="/path/to/node"`` attribute to a config element. Element contents will be substituted with the contents of the /path/to/node ZooKeeper node. The ZooKeeper node can contain a whole XML subtree, and it will be inserted as a child of the substituted node. - -The 'config.xml' file can specify a separate config with user settings, profiles, and quotas. The relative path to this config is set in the 'users_config' element. By default, it is 'users.xml'. If 'users_config' is omitted, the user settings, profiles, and quotas are specified directly in ``config.xml``. For ``users_config``, overrides and substitutions may also exist in files from the ``users_config.d`` directory (for example, ``users.d``). - -For each config file, the server also generates ``file-preprocessed.xml`` files on launch. These files contain all the completed substitutions and overrides, and they are intended for informational use. If ZooKeeper substitutions were used in a config file and the ZooKeeper is unavailable during server startup, the configuration is loaded from the respective preprocessed file. - -The server tracks changes to config files and files and ZooKeeper nodes that were used for substitutions and overrides and reloads users and clusters configurations in runtime. That is, you can add or change users, clusters and their settings without relaunching the server. diff --git a/docs/en/data_types/index.rst b/docs/en/operations/index.md similarity index 60% rename from docs/en/data_types/index.rst rename to docs/en/operations/index.md index 97a3007e2bf..d7621689769 100644 --- a/docs/en/data_types/index.rst +++ b/docs/en/operations/index.md @@ -1,8 +1,10 @@ -Data types -========== +# Operation +```eval_rst .. toctree:: :glob: * */index +``` + diff --git a/docs/en/operations/index.rst b/docs/en/operations/index.rst deleted file mode 100644 index 2dd3b745e81..00000000000 --- a/docs/en/operations/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -Operations -========== - -.. toctree:: - :glob: - - * - */index diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md new file mode 100644 index 00000000000..d7b1a61ce7f --- /dev/null +++ b/docs/en/operations/quotas.md @@ -0,0 +1,101 @@ +# Quotas + +Quotas allow you to limit resource usage over a period of time, or simply track the use of resources. +Quotas are set up in the user config. This is usually 'users.xml'. + +The system also has a feature for limiting the complexity of a single query. See the section "Restrictions on query complexity"). + +In contrast to query complexity restrictions, quotas: + +- Place restrictions on a set of queries that can be run over a period of time, instead of limiting a single query. +- Account for resources spent on all remote servers for distributed query processing. + +Let's look at the section of the 'users.xml' file that defines quotas. + +```xml + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + +``` + +By default, the quota just tracks resource consumption for each hour, without limiting usage. +The resource consumption calculated for each interval is output to the server log after each request. + +```xml + + + + + 3600 + + 1000 + 100 + 1000000000 + 100000000000 + 900 + + + + 86400 + + 10000 + 1000 + 5000000000 + 500000000000 + 7200 + + +``` + +For the 'statbox' quota, restrictions are set for every hour and for every 24 hours (86,400 seconds). The time interval is counted starting from an implementation-defined fixed moment in time. In other words, the 24-hour interval doesn't necessarily begin at midnight. + +When the interval ends, all collected values are cleared. For the next hour, the quota calculation starts over. + +Here are the amounts that can be restricted: + +`queries` – The total number of requests. + +`errors` – The number of queries that threw an exception. + +`result_rows` – The total number of rows given as the result. + +`read_rows` – The total number of source rows read from tables for running the query, on all remote servers. + +`execution_time` – The total query execution time, in seconds (wall time). + +If the limit is exceeded for at least one time interval, an exception is thrown with a text about which restriction was exceeded, for which interval, and when the new interval begins (when queries can be sent again). + +Quotas can use the "quota key" feature in order to report on resources for multiple keys independently. Here is an example of this: + +```xml + + + + +``` + +The quota is assigned to users in the 'users' section of the config. See the section "Access rights". + +For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over". + +When the server is restarted, quotas are reset. + diff --git a/docs/en/operations/quotas.rst b/docs/en/operations/quotas.rst deleted file mode 100644 index 6b7be9b0e49..00000000000 --- a/docs/en/operations/quotas.rst +++ /dev/null @@ -1,95 +0,0 @@ -Quotas -====== - -Quotas allow you to limit resource usage over a period of time, or simply track the use of resources. -Quotas are set up in the user config. This is usually ``users.xml``. - -The system also has a feature for limiting the complexity of a single query (see the section "Restrictions on query complexity"). - -In contrast to query complexity restrictions, quotas: - * place restrictions on a set of queries that can be run over a period of time, instead of limiting a single query. - * account for resources spent on all remote servers for distributed query processing. - -Let's look at the section of the ``users.xml`` file that defines quotas. - -.. code-block:: xml - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - -By default, the quota just tracks resource consumption for each hour, without limiting usage. - -.. code-block:: xml - - - - - - 3600 - 1000 - 100 - 1000000000 - 100000000000 - 900 - - - 86400 - 10000 - 1000 - 5000000000 - 500000000000 - 7200 - - - -For the ``statbox`` quota, restrictions are set for every hour and for every 24 hours (86,400 seconds). The time interval is counted starting from an implementation-defined fixed moment in time. In other words, the 24-hour interval doesn't necessarily begin at midnight. - -When the interval ends, all collected values are cleared. For the next hour, the quota calculation starts over. - -Let's examine the amounts that can be restricted: - -``queries`` - The overall number of queries. - -``errors`` - The number of queries that threw exceptions. - -``result_rows`` - The total number of rows output in results. - -``read_rows`` - The total number of source rows retrieved from tables for running a query, on all remote servers. - -``execution_time`` - The total time of query execution, in seconds (wall time). - -If the limit is exceeded for at least one time interval, an exception is thrown with a text about which restriction was exceeded, for which interval, and when the new interval begins (when queries can be sent again). - -Quotas can use the "quota key" feature in order to report on resources for multiple keys independently. Here is an example of this: - -.. code-block:: xml - - - - - - -The quota is assigned to users in the ``users`` section of the config. See the section "Access rights". - -For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over". - -When the server is restarted, quotas are reset. diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md new file mode 100644 index 00000000000..365a23a7022 --- /dev/null +++ b/docs/en/operations/server_settings/index.md @@ -0,0 +1,19 @@ + + +# Server configuration parameters + +This section contains descriptions of server settings that cannot be changed at the session or query level. + +These settings are stored in the ` config.xml` file on the ClickHouse server. + +Other settings are described in the "[Settings](../settings/index.md#settings)" section. + +Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes). + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md new file mode 100644 index 00000000000..afe4be482fb --- /dev/null +++ b/docs/en/operations/server_settings/settings.md @@ -0,0 +1,705 @@ +# Server settings + + + +## builtin_dictionaries_reload_interval + +The interval in seconds before reloading built-in dictionaries. + +ClickHouse reloads built-in dictionaries every x seconds. This makes it possible to edit dictionaries "on the fly" without restarting the server. + +Default value: 3600. + +**Example** + +```xml +3600 +``` + + + +## compression + +Data compression settings. + +
+ +Don't use it if you have just started using ClickHouse. + +
+ +The configuration looks like this: + +```xml + + + + + ... + +``` + +You can configure multiple sections ``. + +Block field ``: + +- ``min_part_size`` – The minimum size of a table part. +- ``min_part_size_ratio`` – The ratio of the minimum size of a table part to the full size of the table. +- ``method`` – Compression method. Acceptable values ​: ``lz4`` or ``zstd``(experimental). + +ClickHouse checks ` min_part_size` and ` min_part_size_ratio` and processes the ` case` blocks that match these conditions. If none of the `` matches, ClickHouse applies the `lz4` compression algorithm. + +**Example** + +```xml + + + 10000000000 + 0.01 + zstd + + +``` + + + +## default_database + +The default database. + +Use a [ SHOW DATABASES](../query_language/queries.md#query_language_queries_show_databases) query to get a list of databases. + +**Example** + +```xml +default +``` + + + +## default_profile + +Default settings profile. + +Settings profiles are located in the file specified in the [user_config](#server_settings-users_config) parameter. + +**Example** + +```xml +default +``` + + + +## dictionaries_config + +The path to the config file for external dictionaries. + +Path: + +- Specify the absolute path or the path relative to the server config file. +- The path can contain wildcards \* and ?. + +See also "[External dictionaries](../../dicts/external_dicts.md#dicts-external_dicts)". + +**Example** + +```xml +*_dictionary.xml +``` + + + +## dictionaries_lazy_load + +Lazy loading of dictionaries. + +If ` true`, then each dictionary is created on first use. If dictionary creation failed, the function that was using the dictionary throws an exception. + +If `false`, all dictionaries are created when the server starts, and if there is an error, the server shuts down. + +The default is ` true`. + +**Example** + +```xml +true +``` + + + +## format_schema_path + +The path to the directory with the schemas for the input data, such as schemas for the [ CapnProto](../../formats/capnproto.md#format_capnproto) format. + +**Example** + +```xml + + format_schemas/ +``` + + + +## graphite + +Sending data to [Graphite](https://github.com/graphite-project). + +Settings: + +- host – The Graphite server. +- port – The port on the Graphite server. +- interval – The interval for sending, in seconds. +- timeout – The timeout for sending data, in seconds. +- root_path – Prefix for keys. +- metrics – Sending data from a :ref:`system_tables-system.metrics` table. +- events – Sending data from a :ref:`system_tables-system.events` table. +- asynchronous_metrics – Sending data from a :ref:`system_tables-system.asynchronous_metrics` table. + +You can configure multiple `` clauses. For instance, you can use this for sending different data at different intervals. + +**Example** + +```xml + + localhost + 42000 + 0.1 + 60 + one_min + true + true + true + +``` + + + +## graphite_rollup + +Settings for thinning data for Graphite. + +For more details, see [ GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree). + +**Example** + +```xml + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + + + +## http_port/https_port + +The port for connecting to the server over HTTP(s). + +If `https_port` is specified, [openSSL](#server_settings-openSSL) must be configured. + +If `http_port` is specified, the openSSL configuration is ignored even if it is set. + +**Example** + +```xml +0000 +``` + + + +## http_server_default_response + +The page that is shown by default when you access the ClickHouse HTTP(s) server. + +**Example** + +Opens `https://tabix.io/` when accessing ` http://localhost: http_port`. + +```xml + +
]]> + +``` + + + +## include_from + +The path to the file with substitutions. + +For details, see the section "[Configuration files](../configuration_files.md#configuration_files)". + +**Example** + +```xml +/etc/metrica.xml +``` + + + +## interserver_http_port + +Port for exchanging data between ClickHouse servers. + +**Example** + +```xml +9009 +``` + + + +## interserver_http_host + +The host name that can be used by other servers to access this server. + +If omitted, it is defined in the same way as the ` hostname-f` command. + +Useful for breaking away from a specific network interface. + +**Example** + +```xml +example.yandex.ru +``` + + + +## keep_alive_timeout + +The number of milliseconds that ClickHouse waits for incoming requests before closing the connection. + +**Example** + +```xml +3 +``` + + + +## listen_host + +Restriction on hosts that requests can come from. If you want the server to answer all of them, specify `::`. + +Examples: + +```xml +::1127.0.0.1 +``` + + + +## logger + +Logging settings. + +Keys: + +- level – Logging level. Acceptable values: ``trace``, ``debug``, ``information``, ``warning``, ``error``. +- log – The log file. Contains all the entries according to `` level``. +- errorlog – Error log file. +- size – Size of the file. Applies to ``log``and``errorlog``. Once the file reaches ``size``, ClickHouse archives and renames it, and creates a new log file in its place. +- count – The number of archived log files that ClickHouse stores. + +**Example** + +```xml + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + +``` + + + +## macros + +Parameter substitutions for replicated tables. + +Can be omitted if replicated tables are not used. + +For more information, see the section "[Creating replicated tables](../../table_engines/replication.md#table_engines-replication-creation_of_rep_tables)". + +**Example** + +```xml + +``` + + + +## mark_cache_size + +Approximate size (in bytes) of the cache of "marks" used by [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) engines. + +The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120. + +**Example** + +```xml +5368709120 +``` + + + +## max_concurrent_queries + +The maximum number of simultaneously processed requests. + +**Example** + +```xml +100 +``` + + + +## max_connections + +The maximum number of inbound connections. + +**Example** + +```xml +4096 +``` + + + +## max_open_files + +The maximum number of open files. + +By default: `maximum`. + +We recommend using this option in Mac OS X, since the ` getrlimit()` function returns an incorrect value. + +**Example** + +```xml +262144 +``` + + + +## max_table_size_to_drop + +Restriction on deleting tables. + +If the size of a [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. + +If you still need to delete the table without restarting the ClickHouse server, create the ` /flags/force_drop_table` file and run the DROP query. + +Default value: 50 GB. + +The value 0 means that you can delete all tables without any restrictions. + +**Example** + +```xml +0 +``` + + + +## merge_tree + +Fine tuning for tables in the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. + +For more information, see the MergeTreeSettings.h header file. + +**Example** + +```xml + + 5 + +``` + + + +## openSSL + +SSL client/server configuration. + +Support for SSL is provided by the `` libpoco`` library. The description of the interface is in the [ SSLManager.h file.](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) + +Keys for server/client settings: + +- privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. +- certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `` privateKeyFile`` contains the certificate. +- caConfig – The path to the file or directory that contains trusted root certificates. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Acceptable values: ``none``, ``relaxed``, ``strict``, ``once``. +- verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. +- loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `` true``, `` false``. | +- cipherList - Поддерживаемые OpenSSL-шифры. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``. +- cacheSessions – Enables or disables caching sessions. Must be used in combination with ``sessionIdContext``. Acceptable values: `` true``, `` false``. +- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed ``SSL_MAX_SSL_SESSION_ID_LENGTH``. This parameter is always recommended, since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: ``${application.name}``. +- sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. +- sessionTimeout – Time for caching the session on the server. +- extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `` true``, `` false``. +- requireTLSv1 – Require a TLSv1 connection. Acceptable values: `` true``, `` false``. +- requireTLSv1_1 – Require a TLSv1.1 connection. Acceptable values: `` true``, `` false``. +- requireTLSv1 – Require a TLSv1.2 connection. Acceptable values: `` true``, `` false``. +- fips – Activates OpenSSL FIPS mode. Supported if the library's OpenSSL version supports FIPS. +- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ````, ``KeyFileHandler``, ``test``, ````. +- invalidCertificateHandler – Class (subclass of CertificateHandler) for verifying invalid certificates. For example: `` ConsoleCertificateHandler `` . +- disableProtocols – Protocols that are not allowed to use. +- preferServerCiphers – Preferred server ciphers on the client. + +**Example of settings:** + +```xml + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + +``` + + + +## part_log + +Logging events that are associated with the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data type. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. + +Queries are logged in the ClickHouse table, not in a separate file. + +Columns in the log: + +- event_time – Date of the event. +- duration_ms – Duration of the event. +- event_type – Type of event. 1 – new data part; 2 – merge result; 3 – data part downloaded from replica; 4 – data part deleted. +- database_name – The name of the database. +- table_name – Name of the table. +- part_name – Name of the data part. +- size_in_bytes – Size of the data part in bytes. +- merged_from – An array of names of data parts that make up the merge (also used when downloading a merged part). +- merge_time_ms – Time spent on the merge. + +Use the following parameters to configure logging: + +- database – Name of the database. +- table – Name of the table. +- flush_interval_milliseconds – Interval for flushing data from memory to the disk. + +**Example** + +```xml + + system + part_log
+ 7500 + +``` + + + +## path + +The path to the directory containing data. + +
+ +The end slash is mandatory. + +
+ +**Example** + +```xml +/var/lib/clickhouse/ +``` + + + +## query_log + +Setting for logging queries received with the [log_queries=1](#settings-log_queries) setting. + +Queries are logged in the ClickHouse table, not in a separate file. + +Use the following parameters to configure logging: + +- database – Name of the database. +- table – Name of the table. +- flush_interval_milliseconds – Interval for flushing data from memory to the disk. + +If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. + +**Example** + +```xml + + system + query_log
+ 7500 + +``` + + + +## remote_servers + +Configuration of clusters used by the Distributed table engine. + +For more information, see the section "[Duplicated table engine](../../table_engines/distributed.md#table_engines-distributed)". + +**Example** + +```xml + +``` + +For the value of the `incl` attribute, see the section "[Configuration files](../configuration_files.md#configuration_files)". + + + +## resharding + +The ZooKeeper path to the task queue. + +For more information, see "[Resharding](../../table_engines/resharding.md#table_engines-resharding)". + +**Example** + +```xml + + /clickhouse/task_queue + +``` + + + +## timezone + +The server's time zone. + +Specified as an IANA identifier for the UTC time zone or geographic location (for example, Africa/Abidjan). + +The time zone is necessary for conversions between String and DateTime formats when DateTime fields are output to text format (printed on the screen or in a file), and when getting DateTime from a string. In addition, the time zone is used in functions that work with the time and date if they didn't receive the time zone in the input parameters. + +**Example** + +```xml +Europe/Moscow +``` + + + +## tcp_port + +Port for communicating with clients over the TCP protocol. + +**Example** + +```xml +9000 +``` + + + +## tmp_path + +Path to temporary data for processing large queries. + +
+ +The end slash is mandatory. + +
+ +**Example** + +```xml +/var/lib/clickhouse/tmp/ +``` + + + +## uncompressed_cache_size + +Cache size (in bytes) for uncompressed data used by table engines from the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. + +There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#settings-use_uncompressed_cache) is enabled. + +The uncompressed cache is advantageous for very short queries in individual cases. + +**Example** + +```xml +8589934592 +``` + + + +## users_config + +Path to the file that contains: + +- User configurations. +- Access rights. +- Settings profiles. +- Quota settings. + +**Example** + +```xml +users.xml +``` + + + +## zookeeper + +Configuration of ZooKeeper servers. + +ClickHouse uses ZooKeeper for storing replica metadata when using replicated tables. + +This parameter can be omitted if replicated tables are not used. + +For more information, see the section "[Replication](../../table_engines/replication.md#table_engines-replication)". + +**Example** + +```xml + +``` + diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md new file mode 100644 index 00000000000..70c660bbb9e --- /dev/null +++ b/docs/en/operations/settings/index.md @@ -0,0 +1,31 @@ + + +# Settings + +There are multiple ways to make all the settings described below. +Settings are configured in layers, so each subsequent layer redefines the previous settings. + +Ways to configure settings, in order of priority: + +- Settings in the server config file. + + Set via user profiles. + +- For the session. + + Send ` SET setting=value` from the ClickHouse console client in interactive mode. +Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter. + +- For a query. + - When starting the ClickHouse console client in non-interactive mode, set the startup parameter `--setting=value`. + - When using the HTTP API, pass CGI parameters (`URL?setting_1=value&setting_2=value...`). + +Settings that can only be made in the server config file are not covered in this section. + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/operations/settings/index.rst b/docs/en/operations/settings/index.rst deleted file mode 100644 index 73b6779d8c4..00000000000 --- a/docs/en/operations/settings/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -Settings -======== - -In this section, we review settings that you can make using a SET query or in a config file. -Remember that these settings can be set for a session or globally. -Settings that can only be made in the server config file are not covered here. - - -There are three ways to setup settings (sorted by priority): - -* Settings in server configuration files. - - They are set in user profiles. - -* Session-level settings. - - Use ``SET setting=value`` command in ClickHouse console client running in interactive mode. - Also sessions are supported in HTTP interface (you need to pass ``session_id`` HTTP parameter) - -* Query-level settings. - - Use ``--setting=value`` command line parameters in non-iteractive mode of ClickHouse console client. - Use HTTP parameters (``URL?setting_1=value&setting_2=value...``) with HTTP ClickHouse interface. - - -.. toctree:: - :glob: - - * diff --git a/docs/en/operations/settings/query_complexity.rst b/docs/en/operations/settings/query_complexity.md similarity index 54% rename from docs/en/operations/settings/query_complexity.rst rename to docs/en/operations/settings/query_complexity.md index 0a2dc836262..627c2b00ea1 100644 --- a/docs/en/operations/settings/query_complexity.rst +++ b/docs/en/operations/settings/query_complexity.md @@ -1,32 +1,33 @@ -Restrictions on query complexity -================================ +# Restrictions on query complexity + Restrictions on query complexity are part of the settings. They are used in order to provide safer execution from the user interface. -Almost all the restrictions only apply to SELECTs. -For distributed query processing, restrictions are applied on each server separately. +Almost all the restrictions only apply to SELECTs.For distributed query processing, restrictions are applied on each server separately. Restrictions on the "maximum amount of something" can take the value 0, which means "unrestricted". Most restrictions also have an 'overflow_mode' setting, meaning what to do when the limit is exceeded. -It can take one of two values: 'throw' or 'break'. Restrictions on aggregation (``group_by_overflow_mode``) also have the value ``any``. +It can take one of two values: `throw` or `break`. Restrictions on aggregation (group_by_overflow_mode) also have the value `any`. -``throw`` - Throw an exception (default). +`throw` – Throw an exception (default). -``break`` - Stop executing the query and return the partial result, as if the source data ran out. +`break` – Stop executing the query and return the partial result, as if the source data ran out. -``any`` (only for group_by_overflow_mode) - Continuing aggregation for the keys that got into the set, but don't add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don't add new keys to the set. -readonly --------- -If set to 0, allows to run any queries. -If set to 1, allows to run only queries that don't change data or settings (e.g. SELECT or SHOW). INSERT and SET are forbidden. -If set to 2, allows to run queries that don't change data (SELECT, SHOW) and allows to change settings (SET). + -After you set the read-only mode, you won't be able to disable it in the current session. +## readonly + +With a value of 0, you can execute any queries. +With a value of 1, you can only execute read requests (such as SELECT and SHOW). Requests for writing and changing settings (INSERT, SET) are prohibited. +With a value of 2, you can process read queries (SELECT, SHOW) and change settings (SET). + +After enabling readonly mode, you can't disable it in the current session. When using the GET method in the HTTP interface, 'readonly = 1' is set automatically. In other words, for queries that modify data, you can only use the POST method. You can send the query itself either in the POST body, or in the URL parameter. -max_memory_usage ----------------- +## max_memory_usage + The maximum amount of memory consumption when running a query on a single server. By default, 10 GB. The setting doesn't consider the volume of available memory or the total volume of memory on the machine. @@ -35,139 +36,142 @@ You can use SHOW PROCESSLIST to see the current memory consumption for each quer In addition, the peak memory consumption is tracked for each query and written to the log. Certain cases of memory consumption are not tracked: - * Large constants (for example, a very long string constant). - * The states of 'groupArray' aggregate functions, and also 'quantile' (it is tracked for 'quantileTiming'). -Memory consumption is not fully considered for aggregate function states ``min``, ``max``, ``any``, ``anyLast``, ``argMin``, and ``argMax`` from String and Array arguments. +- Large constants (for example, a very long string constant). +- The states of certain aggregate functions. + +Memory consumption is not fully considered for aggregate function states 'min', 'max', 'any', 'anyLast', 'argMin', and 'argMax' from String and Array arguments. + +## max_rows_to_read -max_rows_to_read ----------------- The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little. When running a query in multiple threads, the following restrictions apply to each thread separately. Maximum number of rows that can be read from a table when running a query. -max_bytes_to_read ------------------ +## max_bytes_to_read + Maximum number of bytes (uncompressed data) that can be read from a table when running a query. -read_overflow_mode ------------------- -What to do when the volume of data read exceeds one of the limits: ``throw`` or ``break``. ``By default, throw``. +## read_overflow_mode + +What to do when the volume of data read exceeds one of the limits: 'throw' or 'break'. By default, throw. + +## max_rows_to_group_by -max_rows_to_group_by --------------------- Maximum number of unique keys received from aggregation. This setting lets you limit memory consumption when aggregating. -group_by_overflow_mode ----------------------- -What to do when the number of unique keys for aggregation exceeds the limit: ``throw``, ``break``, or ``any``. ``By default, throw``. +## group_by_overflow_mode + +What to do when the number of unique keys for aggregation exceeds the limit: 'throw', 'break', or 'any'. By default, throw. Using the 'any' value lets you run an approximation of GROUP BY. The quality of this approximation depends on the statistical nature of the data. -max_rows_to_sort ----------------- +## max_rows_to_sort + Maximum number of rows before sorting. This allows you to limit memory consumption when sorting. -max_bytes_to_sort ------------------ +## max_bytes_to_sort + Maximum number of bytes before sorting. -sort_overflow_mode ------------------- -What to do if the number of rows received before sorting exceeds one of the limits: ``throw`` or ``break``. ``By default, throw``. +## sort_overflow_mode + +What to do if the number of rows received before sorting exceeds one of the limits: 'throw' or 'break'. By default, throw. + +## max_result_rows -max_result_rows ---------------- Limit on the number of rows in the result. Also checked for subqueries, and on remote servers when running parts of a distributed query. -max_result_bytes ----------------- +## max_result_bytes + Limit on the number of bytes in the result. The same as the previous setting. -result_overflow_mode --------------------- -What to do if the volume of the result exceeds one of the limits: ``throw`` or ``break``. By default, throw. -Using ``break`` is similar to using ``LIMIT``. +## result_overflow_mode + +What to do if the volume of the result exceeds one of the limits: 'throw' or 'break'. By default, throw. +Using 'break' is similar to using LIMIT. + +## max_execution_time -max_execution_time ------------------- Maximum query execution time in seconds. At this time, it is not checked for one of the sorting stages, or when merging and finalizing aggregate functions. -timeout_overflow_mode ---------------------- -What to do if the query is run longer than ``max_execution_time``: ``throw`` or ``break``. ``By default, throw``. +## timeout_overflow_mode -min_execution_speed -------------------- -Minimal execution speed in rows per second. Checked on every data block when ``timeout_before_checking_execution_speed`` expires. If the execution speed is lower, an exception is thrown. +What to do if the query is run longer than 'max_execution_time': 'throw' or 'break'. By default, throw. -timeout_before_checking_execution_speed ---------------------------------------- -Checks that execution speed is not too slow (no less than ``min_execution_speed``), after the specified time in seconds has expired. +## min_execution_speed + +Minimal execution speed in rows per second. Checked on every data block when 'timeout_before_checking_execution_speed' expires. If the execution speed is lower, an exception is thrown. + +## timeout_before_checking_execution_speed + +Checks that execution speed is not too slow (no less than 'min_execution_speed'), after the specified time in seconds has expired. + +## max_columns_to_read -max_columns_to_read -------------------- Maximum number of columns that can be read from a table in a single query. If a query requires reading a greater number of columns, it throws an exception. -max_temporary_columns ---------------------- +## max_temporary_columns + Maximum number of temporary columns that must be kept in RAM at the same time when running a query, including constant columns. If there are more temporary columns than this, it throws an exception. -max_temporary_non_const_columns -------------------------------- +## max_temporary_non_const_columns + The same thing as 'max_temporary_columns', but without counting constant columns. Note that constant columns are formed fairly often when running a query, but they require approximately zero computing resources. -max_subquery_depth ------------------- -Maximum nesting depth of subqueries. If subqueries are deeper, an exception is thrown. ``By default, 100``. +## max_subquery_depth + +Maximum nesting depth of subqueries. If subqueries are deeper, an exception is thrown. By default, 100. + +## max_pipeline_depth -max_pipeline_depth ------------------- Maximum pipeline depth. Corresponds to the number of transformations that each data block goes through during query processing. Counted within the limits of a single server. If the pipeline depth is greater, an exception is thrown. By default, 1000. -max_ast_depth -------------- -Maximum nesting depth of a query syntactic tree. If exceeded, an exception is thrown. At this time, it isn't checked during parsing, but only after parsing the query. That is, a syntactic tree that is too deep can be created during parsing, but the query will fail. By default, 1000. +## max_ast_depth + +Maximum nesting depth of a query syntactic tree. If exceeded, an exception is thrown. +At this time, it isn't checked during parsing, but only after parsing the query. That is, a syntactic tree that is too deep can be created during parsing, but the query will fail. By default, 1000. + +## max_ast_elements -max_ast_elements ----------------- Maximum number of elements in a query syntactic tree. If exceeded, an exception is thrown. -In the same way as the previous setting, it is checked only after parsing the query. ``By default, 10,000``. +In the same way as the previous setting, it is checked only after parsing the query. By default, 10,000. + +## max_rows_in_set -max_rows_in_set ---------------- Maximum number of rows for a data set in the IN clause created from a subquery. -max_bytes_in_set ----------------- +## max_bytes_in_set + Maximum number of bytes (uncompressed data) used by a set in the IN clause created from a subquery. -set_overflow_mode ------------------ -What to do when the amount of data exceeds one of the limits: ``throw`` or ``break``. ``By default, throw``. +## set_overflow_mode + +What to do when the amount of data exceeds one of the limits: 'throw' or 'break'. By default, throw. + +## max_rows_in_distinct -max_rows_in_distinct --------------------- Maximum number of different rows when using DISTINCT. -max_bytes_in_distinct ---------------------- +## max_bytes_in_distinct + Maximum number of bytes used by a hash table when using DISTINCT. -distinct_overflow_mode ----------------------- -What to do when the amount of data exceeds one of the limits: ``throw`` or ``break``. ``By default, throw``. +## distinct_overflow_mode + +What to do when the amount of data exceeds one of the limits: 'throw' or 'break'. By default, throw. + +## max_rows_to_transfer -max_rows_to_transfer --------------------- Maximum number of rows that can be passed to a remote server or saved in a temporary table when using GLOBAL IN. -max_bytes_to_transfer ---------------------- +## max_bytes_to_transfer + Maximum number of bytes (uncompressed data) that can be passed to a remote server or saved in a temporary table when using GLOBAL IN. -transfer_overflow_mode ----------------------- -What to do when the amount of data exceeds one of the limits: ``throw`` or ``break``. ``By default, throw``. +## transfer_overflow_mode + +What to do when the amount of data exceeds one of the limits: 'throw' or 'break'. By default, throw. + diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md new file mode 100644 index 00000000000..278a6dd5dd1 --- /dev/null +++ b/docs/en/operations/settings/settings.md @@ -0,0 +1,352 @@ +# Settings + + + +## distributed_product_mode + +Alters the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables. + +ClickHouse applies the configuration if the subqueries on any level have a distributed table that exists on the local server and has more than one shard. + +Restrictions: + +- Only applied for IN and JOIN subqueries. +- Used only if a distributed table is used in the FROM clause. +- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote)function. + +Possible values: + + + +## fallback_to_stale_replicas_for_distributed_queries + +Forces a query to an out-of-date replica if updated data is not available. See "[Replication](../../table_engines/replication.md#table_engines-replication)". + +ClickHouse selects the most relevant from the outdated replicas of the table. + +Used when performing ` SELECT` from a distributed table that points to replicated tables. + +By default, 1 (enabled). + + + +## force_index_by_date + +Disables query execution if the index can't be used by date. + +Works with tables in the MergeTree family. + +If `force_index_by_date=1`, ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". + + + +## force_primary_key + +Disables query execution if indexing by the primary key is not possible. + +Works with tables in the MergeTree family. + +If `force_primary_key=1`, ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". + + + +## fsync_metadata + +Enable or disable fsync when writing .sql files. By default, it is enabled. + +It makes sense to disable it if the server has millions of tiny table chunks that are constantly being created and destroyed. + +## input_format_allow_errors_num + +Sets the maximum number of acceptable errors when reading from text formats (CSV, TSV, etc.). + +The default value is 0. + +Always pair it with `input_format_allow_errors_ratio`. To skip errors, both settings must be greater than 0. + +If an error occurred while reading rows but the error counter is still less than `input_format_allow_errors_num`, ClickHouse ignores the row and moves on to the next one. + +If `input_format_allow_errors_num`is exceeded, ClickHouse throws an exception. + +## input_format_allow_errors_ratio + +Sets the maximum percentage of errors allowed when reading from text formats (CSV, TSV, etc.). +The percentage of errors is set as a floating-point number between 0 and 1. + +The default value is 0. + +Always pair it with `input_format_allow_errors_num`. To skip errors, both settings must be greater than 0. + +If an error occurred while reading rows but the error counter is still less than `input_format_allow_errors_ratio`, ClickHouse ignores the row and moves on to the next one. + +If `input_format_allow_errors_ratio` is exceeded, ClickHouse throws an exception. + +## max_block_size + +In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. `max_block_size` is a recommendation for what size of block (in number of rows) to load from tables. The block size shouldn't be too small, so that the expenditures on each block are still noticeable, but not too large, so that the query with LIMIT that is completed after the first block is processed quickly, so that too much memory isn't consumed when extracting a large number of columns in multiple threads, and so that at least some cache locality is preserved. + +By default, 65,536. + +Blocks the size of `max_block_size` are not always loaded from the table. If it is obvious that less data needs to be retrieved, a smaller block is processed. + +## preferred_block_size_bytes + +Used for the same purpose as `max_block_size`, but it sets the recommended block size in bytes by adapting it to the number of rows in the block. +However, the block size cannot be more than `max_block_size` rows. +Disabled by default (set to 0). It only works when reading from MergeTree engines. + + + +## log_queries + +Setting up query the logging. + +Queries sent to ClickHouse with this setup are logged according to the rules in the [query_log](../server_setings/settings.md#server_settings-query_log) server configuration parameter. + +**Example**: + + log_queries=1 + + + +## max_insert_block_size + +The size of blocks to form for insertion into a table. +This setting only applies in cases when the server forms the blocks. +For example, for an INSERT via the HTTP interface, the server parses the data format and forms blocks of the specified size. +But when using clickhouse-client, the client parses the data itself, and the 'max_insert_block_size' setting on the server doesn't affect the size of the inserted blocks. +The setting also doesn't have a purpose when using INSERT SELECT, since data is inserted using the same blocks that are formed after SELECT. + +By default, it is 1,048,576. + +This is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allows sorting more data in RAM. + + + +## max_replica_delay_for_distributed_queries + +Disables lagging replicas for distributed queries. See "[Replication](../../table_engines/replication.md#table_engines-replication)". + +Sets the time in seconds. If a replica lags more than the set value, this replica is not used. + +Default value: 0 (off). + +Used when performing ` SELECT` from a distributed table that points to replicated tables. + +## max_threads + +The maximum number of query processing threads + +- excluding threads for retrieving data from remote servers (see the 'max_distributed_connections' parameter). + +This parameter applies to threads that perform the same stages of the query processing pipeline in parallel. +For example, if reading from a table, evaluating expressions with functions, filtering with WHERE and pre-aggregating for GROUP BY can all be done in parallel using at least 'max_threads' number of threads, then 'max_threads' are used. + +By default, 8. + +If less than one SELECT query is normally run on a server at a time, set this parameter to a value slightly less than the actual number of processor cores. + +For queries that are completed quickly because of a LIMIT, you can set a lower 'max_threads'. For example, if the necessary number of entries are located in every block and max_threads = 8, 8 blocks are retrieved, although it would have been enough to read just one. + +The smaller the `max_threads` value, the less memory is consumed. + +## max_compress_block_size + +The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). If the size is reduced, the compression rate is significantly reduced, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. There usually isn't any reason to change this setting. + +Don't confuse blocks for compression (a chunk of memory consisting of bytes) and blocks for query processing (a set of rows from a table). + +## min_compress_block_size + +For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536. + +The actual size of the block, if the uncompressed data is less than 'max_compress_block_size', is no less than this value and no less than the volume of data for one mark. + +Let's look at an example. Assume that 'index_granularity' was set to 8192 during table creation. + +We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, the total will be 32 KB of data. Since min_compress_block_size = 65,536, a compressed block will be formed for every two marks. + +We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won't be decompressed. + +There usually isn't any reason to change this setting. + +## max_query_size + +The maximum part of a query that can be taken to RAM for parsing with the SQL parser. +The INSERT query also contains data for INSERT that is processed by a separate stream parser (that consumes O(1) RAM), which is not included in this restriction. + +The default is 256 KiB. + +## interactive_delay + +The interval in microseconds for checking whether request execution has been canceled and sending the progress. + +By default, 100,000 (check for canceling and send progress ten times per second). + +## connect_timeout + +## receive_timeout + +## send_timeout + +Timeouts in seconds on the socket used for communicating with the client. + +By default, 10, 300, 300. + +## poll_interval + +Lock in a wait loop for the specified number of seconds. + +By default, 10. + +## max_distributed_connections + +The maximum number of simultaneous connections with remote servers for distributed processing of a single query to a single Distributed table. We recommend setting a value no less than the number of servers in the cluster. + +By default, 100. + +The following parameters are only used when creating Distributed tables (and when launching a server), so there is no reason to change them at runtime. + +## distributed_connections_pool_size + +The maximum number of simultaneous connections with remote servers for distributed processing of all queries to a single Distributed table. We recommend setting a value no less than the number of servers in the cluster. + +By default, 128. + +## connect_timeout_with_failover_ms + +The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the 'shard' and 'replica' sections are used in the cluster definition. +If unsuccessful, several attempts are made to connect to various replicas. + +By default, 50. + +## connections_with_failover_max_tries + +The maximum number of connection attempts with each replica, for the Distributed table engine. + +By default, 3. + +## extremes + +Whether to count extreme values (the minimums and maximums in columns of a query result). Accepts 0 or 1. By default, 0 (disabled). +For more information, see the section "Extreme values". + + + +## use_uncompressed_cache + +Whether to use a cache of uncompressed blocks. Accepts 0 or 1. By default, 0 (disabled). +The uncompressed cache (only for tables in the MergeTree family) allows significantly reducing latency and increasing throughput when working with a large number of short queries. Enable this setting for users who send frequent short requests. Also pay attention to the 'uncompressed_cache_size' configuration parameter (only set in the config file) – the size of uncompressed cache blocks. By default, it is 8 GiB. The uncompressed cache is filled in as needed; the least-used data is automatically deleted. + +For queries that read at least a somewhat large volume of data (one million rows or more), the uncompressed cache is disabled automatically in order to save space for truly small queries. So you can keep the 'use_uncompressed_cache' setting always set to 1. + +## replace_running_query + +When using the HTTP interface, the 'query_id' parameter can be passed. This is any string that serves as the query identifier. +If a query from the same user with the same 'query_id' already exists at this time, the behavior depends on the 'replace_running_query' parameter. + +`0` (default) – Throw an exception (don't allow the query to run if a query with the same 'query_id' is already running). + +`1` – Cancel the old query and start running the new one. + +Yandex.Metrica uses this parameter set to 1 for implementing suggestions for segmentation conditions. After entering the next character, if the old query hasn't finished yet, it should be canceled. + +## schema + +This parameter is useful when you are using formats that require a schema definition, such as [ Cap'n Proto](https://capnproto.org/). The value depends on the format. + + + +## stream_flush_interval_ms + +Works for tables with streaming in the case of a timeout, or when a thread generates [ max_insert_block_size](#settings-settings-max_insert_block_size) rows. + +The default value is 7500. + +The smaller the value, the more often data is flushed into the table. Setting the value too low leads to poor performance. + + + +## load_balancing + +Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing. + +### random (default) + +The number of errors is counted for each replica. The query is sent to the replica with the fewest errors, and if there are several of these, to any one of them. +Disadvantages: Server proximity is not accounted for; if the replicas have different data, you will also get different data. + +### nearest_hostname + +The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a host name that is most similar to the server's host name in the config file (for the number of different characters in identical positions, up to the minimum length of both host names). + +For instance, example01-01-1 and example01-01-2.yandex.ru are different in one position, while example01-01-1 and example01-02-2 differ in two places. +This method might seem a little stupid, but it doesn't use external data about network topology, and it doesn't compare IP addresses, which would be complicated for our IPv6 addresses. + +Thus, if there are equivalent replicas, the closest one by name is preferred. +We can also assume that when sending a query to the same server, in the absence of failures, a distributed query will also go to the same servers. So even if different data is placed on the replicas, the query will return mostly the same results. + +### in_order + +Replicas are accessed in the same order as they are specified. The number of errors does not matter. +This method is appropriate when you know exactly which replica is preferable. + +## totals_mode + +How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = 'any' are present. +See the section "WITH TOTALS modifier". + +## totals_auto_threshold + +The threshold for ` totals_mode = 'auto'`. +See the section "WITH TOTALS modifier". + +## default_sample + +Floating-point number from 0 to 1. By default, 1. +Allows you to set the default sampling ratio for all SELECT queries. +(For tables that do not support sampling, it throws an exception.) +If set to 1, sampling is not performed by default. + +## max_parallel_replicas + +The maximum number of replicas for each shard when executing a query. +For consistency (to get different parts of the same data split), this option only works when the sampling key is set. +Replica lag is not controlled. + +## compile + +Enable compilation of queries. By default, 0 (disabled). + +Compilation is only used for part of the query-processing pipeline: for the first stage of aggregation (GROUP BY). +If this portion of the pipeline was compiled, the query may run faster due to deployment of short cycles and inlining aggregate function calls. The maximum performance improvement (up to four times faster in rare cases) is seen for queries with multiple simple aggregate functions. Typically, the performance gain is insignificant. In very rare cases, it may slow down query execution. + +## min_count_to_compile + +How many times to potentially use a compiled chunk of code before running compilation. By default, 3. +If the value is zero, then compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. This can be used for testing; otherwise, use values ​​starting with 1. Compilation normally takes about 5-10 seconds. +If the value is 1 or more, compilation occurs asynchronously in a separate thread. The result will be used as soon as it is ready, including by queries that are currently running. + +Compiled code is required for each different combination of aggregate functions used in the query and the type of keys in the GROUP BY clause. +The results of compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results, since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. + +## input_format_skip_unknown_fields + +If the value is true, running INSERT skips input data from columns with unknown names. Otherwise, this situation will generate an exception. +It works for JSONEachRow and TSKV formats. + +## output_format_json_quote_64bit_integers + +If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes. + + + +## strict_insert_defaults + +Strictly assigns default values when adding data. + +If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert) query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column: + +- `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings. +- `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column. + diff --git a/docs/en/operations/settings/settings.rst b/docs/en/operations/settings/settings.rst deleted file mode 100644 index 6297a3e3373..00000000000 --- a/docs/en/operations/settings/settings.rst +++ /dev/null @@ -1,225 +0,0 @@ -max_block_size --------------- -In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. 'max_block_size' is a recommendation for what size of block (in number of rows) to load from tables. The block size shouldn't be too small, so that the expenditures on each block are still noticeable, but not too large, so that the query with LIMIT that is completed after the first block is processed quickly, so that too much memory isn't consumed when extracting a large number of columns in multiple threads, and so that at least some cache locality is preserved. - -By default, it is 65,536. - -Blocks the size of 'max_block_size' are not always loaded from the table. If it is obvious that less data needs to be retrieved, a smaller block is processed. - -max_insert_block_size ---------------------- -The size of blocks to form for insertion into a table. -This setting only applies in cases when the server forms the blocks. -For example, for an INSERT via the HTTP interface, the server parses the data format and forms blocks of the specified size. -But when using clickhouse-client, the client parses the data itself, and the ``max_insert_block_size`` setting on the server doesn't affect the size of the inserted blocks. -The setting also doesn't have a purpose when using INSERT SELECT, since data is inserted in the same blocks that are formed after SELECT. - -By default, it is 1,048,576. - -This is slightly more than 'max_block_size'. The reason for this is because certain table engines (\*MergeTree) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, \*MergeTree tables sort data during insertion, and a large enough block size allows sorting more data in RAM. - -max_threads ------------ -The maximum number of query processing threads -- excluding threads for retrieving data from remote servers (see the ``max_distributed_connections`` parameter). - -This parameter applies to threads that perform the same stages of the query execution pipeline in parallel. -For example, if reading from a table, evaluating expressions with functions, filtering with WHERE and pre-aggregating for GROUP BY can all be done in parallel using at least ``max_threads`` number of threads, then 'max_threads' are used. - -By default, 8. - -If less than one SELECT query is normally run on a server at a time, set this parameter to a value slightly less than the actual number of processor cores. - -For queries that are completed quickly because of a LIMIT, you can set a lower ``max_threads``. For example, if the necessary number of entries are located in every block and ``max_threads = 8``, 8 blocks are retrieved, although it would have been enough to read just one. - -The smaller the ``max_threads`` value, the less memory is consumed. - -max_compress_block_size ------------------------ -The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, ``1,048,576 (1 MiB)``. If the size is reduced, the compression rate is significantly reduced, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. There usually isn't any reason to change this setting. - -Don't confuse blocks for compression (a chunk of memory consisting of bytes) and blocks for query processing (a set of rows from a table). - -min_compress_block_size ------------------------ -For \*MergeTree tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least ``min_compress_block_size``. By default, 65,536. - -The actual size of the block, if the uncompressed data less than ``max_compress_block_size`` is no less than this value and no less than the volume of data for one mark. - -Let's look at an example. Assume that ``index_granularity`` was set to 8192 during table creation. - -We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, the total will be 32 KB of data. Since ``min_compress_block_size = 65,536``, a compressed block will be formed for every two marks. - -We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won't be decompressed. - -There usually isn't any reason to change this setting. - -max_query_size --------------- -The maximum part of a query that can be taken to RAM for parsing with the SQL parser. -The INSERT query also contains data for INSERT that is processed by a separate stream parser (that consumes O(1) RAM), which is not included in this restriction. - -By default, 256 KiB. - -interactive_delay ------------------ -The interval in microseconds for checking whether request execution has been canceled and sending the progress. - -By default, 100,000 (check for canceling and send progress ten times per second). - -connect_timeout ---------------- - -receive_timeout ---------------- - -send_timeout ------------- -Timeouts in seconds on the socket used for communicating with the client. - -By default, 10, 300, 300. - -poll_interval -------------- -Lock in a wait loop for the specified number of seconds. - -By default, 10. - -max_distributed_connections ---------------------------- -The maximum number of simultaneous connections with remote servers for distributed processing of a single query to a single Distributed table. We recommend setting a value no less than the number of servers in the cluster. - -By default, 100. - -The following parameters are only used when creating Distributed tables (and when launching a server), so there is no reason to change them at runtime. - -distributed_connections_pool_size ---------------------------------- -The maximum number of simultaneous connections with remote servers for distributed processing of all queries to a single Distributed table. We recommend setting a value no less than the number of servers in the cluster. - -By default, 128. - -connect_timeout_with_failover_ms --------------------------------- -The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the 'shard' and 'replica' sections are used in the cluster definition. -If unsuccessful, several attempts are made to connect to various replicas. - -By default, 50. - -connections_with_failover_max_tries ------------------------------------ -The maximum number of connection attempts with each replica, for the Distributed table engine. - -By default, 3. - -extremes --------- -Whether to count extreme values (the minimums and maximums in columns of a query result). -Accepts 0 or 1. By default, 0 (disabled). -For more information, see the section "Extreme values". - -use_uncompressed_cache ----------------------- -Whether to use a cache of uncompressed blocks. Accepts 0 or 1. By default, 0 (disabled). -The uncompressed cache (only for tables in the MergeTree family) allows significantly reducing latency and increasing throughput when working with a large number of short queries. Enable this setting for users who send frequent short requests. Also pay attention to the ``uncompressed_cache_size`` configuration parameter (only set in the config file) - the size of uncompressed cache blocks. -By default, it is 8 GiB. The uncompressed cache is filled in as needed; the least-used data is automatically deleted. - -For queries that read at least a somewhat large volume of data (one million rows or more), the uncompressed cache is disabled automatically in order to save space for truly small queries. So you can keep the ``use_uncompressed_cache`` setting always set to 1. - -replace_running_query ---------------------- -When using the HTTP interface, the 'query_id' parameter can be passed. This is any string that serves as the query identifier. -If a query from the same user with the same 'query_id' already exists at this time, the behavior depends on the 'replace_running_query' parameter. - -``0`` (default) - Throw an exception (don't allow the query to run if a query with the same 'query_id' is already running). -``1`` - Cancel the old query and start running the new one. - -Yandex.Metrica uses this parameter set to 1 for implementing suggestions for segmentation conditions. After entering the next character, if the old query hasn't finished yet, it should be canceled. - -load_balancing --------------- -Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing. - -random (by default) -~~~~~~~~~~~~~~~~~~~ -The number of errors is counted for each replica. The query is sent to the replica with the fewest errors, and if there are several of these, to any one of them. -Disadvantages: Server proximity is not accounted for; if the replicas have different data, you will also get different data. - -nearest_hostname -~~~~~~~~~~~~~~~~ -The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a host name that is most similar to the server's host name in the config file (for the number of different characters in identical positions, up to the minimum length of both host names). - -As an example, example01-01-1 and example01-01-2.yandex.ru are different in one position, while example01-01-1 and example01-02-2 differ in two places. -This method might seem a little stupid, but it doesn't use external data about network topology, and it doesn't compare IP addresses, which would be complicated for our IPv6 addresses. - -Thus, if there are equivalent replicas, the closest one by name is preferred. -We can also assume that when sending a query to the same server, in the absence of failures, a distributed query will also go to the same servers. So even if different data is placed on the replicas, the query will return mostly the same results. - -in_order -~~~~~~~~ -Replicas are accessed in the same order as they are specified. The number of errors does not matter. This method is appropriate when you know exactly which replica is preferable. - -totals_mode ------------ -How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = 'any' are present. -See the section "WITH TOTALS modifier". - -totals_auto_threshold ---------------------- -The threshold for ``totals_mode = 'auto'``. -See the section "WITH TOTALS modifier". - -default_sample --------------- -A floating-point number from 0 to 1. By default, 1. -Allows setting a default sampling coefficient for all SELECT queries. -(For tables that don't support sampling, an exception will be thrown.) -If set to 1, default sampling is not performed. - -max_parallel_replicas ---------------------- -The maximum number of replicas of each shard used when the query is executed. -For consistency (to get different parts of the same partition), this option only works for the specified sampling key. -The lag of the replicas is not controlled. - -compile -------- -Enable query compilation. The default is 0 (disabled). - -Compilation is provided for only part of the request processing pipeline - for the first aggregation step (GROUP BY). -In the event that this part of the pipeline was compiled, the query can work faster, by deploying short loops and inlining the aggregate function calls. The maximum performance increase (up to four times in rare cases) is achieved on queries with several simple aggregate functions. Typically, the performance gain is negligible. In very rare cases, the request may be slowed down. - -min_count_to_compile --------------------- -After how many times, when the compiled piece of code could come in handy, perform its compilation. The default is 3. -In case the value is zero, the compilation is executed synchronously, and the request will wait for the compilation process to finish before continuing. This can be used for testing, otherwise use values ​​starting with 1. Typically, compilation takes about 5-10 seconds. -If the value is 1 or more, the compilation is performed asynchronously, in a separate thread. If the result is ready, it will be immediately used, including those already running at the moment requests. - -The compiled code is required for each different combination of aggregate functions used in the query and the type of keys in GROUP BY. -The compilation results are saved in the build directory as .so files. The number of compilation results is unlimited, since they do not take up much space. When the server is restarted, the old results will be used, except for the server update - then the old results are deleted. - -input_format_skip_unknown_fields --------------------------------- -If the parameter is true, INSERT operation will skip columns with unknown names from input. -Otherwise, an exception will be generated, it is default behavior. -The parameter works only for JSONEachRow and TSKV input formats. - -output_format_json_quote_64bit_integers ---------------------------------------- -If the parameter is true (default value), UInt64 and Int64 numbers are printed as quoted strings in all JSON output formats. -Such behavior is compatible with most JavaScript interpreters that stores all numbers as double-precision floating point numbers. -Otherwise, they are printed as regular numbers. - -stream_flush_interval_ms ------------------------- -This setting only applies in cases when the server forms blocks from streaming table engines. -Either the timeout happens, or the stream produces ``max_insert_block_size`` rows. - -By default, 7500. - -Lower value results in stream flushing to table more often, so the data appears in the destination table faster. -Setting the value too low may result in excessive insertion frequency and lower ingestion efficiency. - -schema ------- -This parameter only applies when used in conjunction with formats requiring a schema definition, for example Cap'n Proto. The parameter value is specific to the format. diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md new file mode 100644 index 00000000000..51ff0e97618 --- /dev/null +++ b/docs/en/operations/settings/settings_profiles.md @@ -0,0 +1,60 @@ +# Settings profiles + +A settings profile is a collection of settings grouped under the same name. Each ClickHouse user has a profile. +To apply all the settings in a profile, set 'profile'. Example: + +```sql +SET profile = 'web' +``` + +- Load the 'web' profile. In other words, set all the options that belong to the 'web' profile. + +Settings profiles are declared in the user config file. This is usually `users.xml`. +Example: + +```xml + + + + + + 8 + + + + + 1000000000 + 100000000000 + + 1000000 + any + + 1000000 + 1000000000 + + 100000 + 100000000 + break + + 600 + 1000000 + 15 + + 25 + 100 + 50 + + 2 + 25 + 50 + 100 + + 1 + + +``` + +The example specifies two profiles: `default` and `web`. The `default` profile has a special purpose: it must always be present and is applied when starting the server. In other words, the 'default' profile contains default settings. The 'web' profile is a regular profile that can be set using the SET query or using a URL parameter in an HTTP query. + +Settings profiles can inherit from each other. To use inheritance, indicate the 'profile' setting before the other settings that are listed in the profile. + diff --git a/docs/en/operations/settings/settings_profiles.rst b/docs/en/operations/settings/settings_profiles.rst deleted file mode 100644 index 7e1f82a55ba..00000000000 --- a/docs/en/operations/settings/settings_profiles.rst +++ /dev/null @@ -1,53 +0,0 @@ -Settings profiles -================= -A settings profile is a collection of settings grouped under the same name. Each ClickHouse user has a profile. -To apply all the settings in a profile, set 'profile'. Example: - -.. code-block:: sql - - SET profile = 'web' - -Load the 'web' profile. That is, set all the options belonging to the 'web' profile. - -Settings profiles are declared in the user config file. This is normally 'users.xml'. - -Example: - -.. code-block:: xml - - - - - - - 8 - - - - 1000000000 - 100000000000 - 1000000 - any - 1000000 - 1000000000 - 100000 - 100000000 - break - 600 - 1000000 - 15 - 25 - 100 - 50 - 2 - 25 - 50 - 100 - 1 - - - - -In the example, two profiles are set: ``default`` and ``web``. The ``default`` profile has a special purpose - it must always be present and is applied when starting the server. In other words, the ``default`` profile contains default settings. The ``web`` profile is a regular profile that can be set using the SET query or using a URL parameter in an HTTP query. - -Settings profiles can inherit from each other. To use inheritance, indicate the 'profile' setting before the other settings that are listed in the profile. diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md new file mode 100644 index 00000000000..11fc8f6da11 --- /dev/null +++ b/docs/en/operations/tips.md @@ -0,0 +1,252 @@ +# Usage recommendations + +## CPU + +The SSE 4.2 instruction set must be supported. Modern processors (since 2008) support it. + +When choosing a processor, prefer a large number of cores and slightly slower clock rate over fewer cores and a higher clock rate. +For example, 16 cores with 2600 MHz is better than 8 cores with 3600 MHz. + +## Hyper-threading + +Don't disable hyper-threading. It helps for some queries, but not for others. + +## Turbo Boost + +Turbo Boost is highly recommended. It significantly improves performance with a typical load. +You can use `turbostat` to view the CPU's actual clock rate under a load. + +## CPU scaling governor + +Always use the `performance` scaling governor. The `on-demand` scaling governor works much worse with constantly high demand. + +```bash +sudo echo 'performance' | tee /sys/devices/system/cpu/cpu\*/cpufreq/scaling_governor +``` + +## CPU limitations + +Processors can overheat. Use `dmesg` to see if the CPU's clock rate was limited due to overheating. +The restriction can also be set externally at the datacenter level. You can use `turbostat` to monitor it under a load. + +## RAM + +For small amounts of data (up to \~200 GB compressed), it is best to use as much memory as the volume of data. +For large amounts of data and when processing interactive (online) queries, you should use a reasonable amount of RAM (128 GB or more) so the hot data subset will fit in the cache of pages. +Even for data volumes of \~50 TB per server, using 128 GB of RAM significantly improves query performance compared to 64 GB. + +## Swap file + +Always disable the swap file. The only reason for not doing this is if you are using ClickHouse on your personal laptop. + +## Huge pages + +Always disable transparent huge pages. It interferes with memory allocators, which leads to significant performance degradation. + +```bash +echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled +``` + +Use `perf top` to watch the time spent in the kernel for memory management. +Permanent huge pages also do not need to be allocated. + +## Storage subsystem + +If your budget allows you to use SSD, use SSD. +If not, use HDD. SATA HDDs 7200 RPM will do. + +Give preference to a lot of servers with local hard drives over a smaller number of servers with attached disk shelves. +But for storing archives with rare queries, shelves will work. + +## RAID + +When using HDD, you can combine their RAID-10, RAID-5, RAID-6 or RAID-50. +For Linux, software RAID is better (with `mdadm`). We don't recommend using LVM. +When creating RAID-10, select the `far` layout. +If your budget allows, choose RAID-10. + +If you have more than 4 disks, use RAID-6 (preferred) or RAID-50, instead of RAID-5. +When using RAID-5, RAID-6 or RAID-50, always increase stripe_cache_size, since the default value is usually not the best choice. + +```bash +echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size +``` + +Calculate the exact number from the number of devices and the block size, using the formula: `2 * num_devices * chunk_size_in_bytes / 4096`. + +A block size of 1025 KB is sufficient for all RAID configurations. +Never set the block size too small or too large. + +You can use RAID-0 on SSD. +Regardless of RAID use, always use replication for data security. + +Enable NCQ with a long queue. For HDD, choose the CFQ scheduler, and for SSD, choose noop. Don't reduce the 'readahead' setting. +For HDD, enable the write cache. + +## File system + +Ext4 is the most reliable option. Set the mount options `noatime, nobarrier`. +XFS is also suitable, but it hasn't been as thoroughly tested with ClickHouse. +Most other file systems should also work fine. File systems with delayed allocation work better. + +## Linux kernel + +Don't use an outdated Linux kernel. In 2015, 3.18.19 was new enough. +Consider using the kernel build from Yandex: – it provides at least a 5% performance increase. + +## Network + +If you are using IPv6, increase the size of the route cache. +The Linux kernel prior to 3.2 had a multitude of problems with IPv6 implementation. + +Use at least a 10 GB network, if possible. 1 Gb will also work, but it will be much worse for patching replicas with tens of terabytes of data, or for processing distributed queries with a large amount of intermediate data. + +## ZooKeeper + +You are probably already using ZooKeeper for other purposes. You can use the same installation of ZooKeeper, if it isn't already overloaded. + +It's best to use a fresh version of ZooKeeper – 3.5 or later. The version in stable Linux distributions may be outdated. + +With the default settings, ZooKeeper is a time bomb: + +> The ZooKeeper server won't delete files from old snapshots and logs when using the default configuration (see autopurge), and this is the responsibility of the operator. + +This bomb must be defused. + +The ZooKeeper (3.5.1) configuration below is used in the Yandex.Metrica production environment as of May 20, 2017: + +zoo.cfg: + +```bash +# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html + +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=30000 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=10 + +maxClientCnxns=2000 + +maxSessionTimeout=60000000 +# the directory where the snapshot is stored. +dataDir=/opt/zookeeper/{{ cluster['name'] }}/data +# Place the dataLogDir to a separate physical disc for better performance +dataLogDir=/opt/zookeeper/{{ cluster['name'] }}/logs + +autopurge.snapRetainCount=10 +autopurge.purgeInterval=1 + + +# To avoid seeks ZooKeeper allocates space in the transaction log file in +# blocks of preAllocSize kilobytes. The default block size is 64M. One reason +# for changing the size of the blocks is to reduce the block size if snapshots +# are taken more often. (Also, see snapCount). +preAllocSize=131072 + +# Clients can submit requests faster than ZooKeeper can process them, +# especially if there are a lot of clients. To prevent ZooKeeper from running +# out of memory due to queued requests, ZooKeeper will throttle clients so that +# there is no more than globalOutstandingLimit outstanding requests in the +# system. The default limit is 1,000.ZooKeeper logs transactions to a +# transaction log. After snapCount transactions are written to a log file a +# snapshot is started and a new transaction log file is started. The default +# snapCount is 10,000. +snapCount=3000000 + +# If this option is defined, requests will be will logged to a trace file named +# traceFile.year.month.day. +#traceFile= + +# Leader accepts client connections. Default value is "yes". The leader machine +# coordinates updates. For higher update throughput at thes slight expense of +# read throughput the leader can be configured to not accept clients and focus +# on coordination. +leaderServes=yes + +standaloneEnabled=false +dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic +``` + +Java version: + +```text +Java(TM) SE Runtime Environment (build 1.8.0_25-b17) +Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) +``` + +JVM parameters: + +```bash +NAME=zookeeper-{{ cluster['name'] }} +ZOOCFGDIR=/etc/$NAME/conf + +# TODO this is really ugly +# How to find out, which jars are needed? +# seems, that log4j requires the log4j.properties file to be in the classpath +CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper" + +ZOOCFG="$ZOOCFGDIR/zoo.cfg" +ZOO_LOG_DIR=/var/log/$NAME +USER=zookeeper +GROUP=zookeeper +PIDDIR=/var/run/$NAME +PIDFILE=$PIDDIR/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +JAVA=/usr/bin/java +ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" +ZOO_LOG4J_PROP="INFO,ROLLINGFILE" +JMXLOCALONLY=false +JAVA_OPTS="-Xms{{ cluster.get('xms','128M') }} \ + -Xmx{{ cluster.get('xmx','1G') }} \ + -Xloggc:/var/log/$NAME/zookeeper-gc.log \ + -XX:+UseGCLogFileRotation \ + -XX:NumberOfGCLogFiles=16 \ + -XX:GCLogFileSize=16M \ + -verbose:gc \ + -XX:+PrintGCTimeStamps \ + -XX:+PrintGCDateStamps \ + -XX:+PrintGCDetails + -XX:+PrintTenuringDistribution \ + -XX:+PrintGCApplicationStoppedTime \ + -XX:+PrintGCApplicationConcurrentTime \ + -XX:+PrintSafepointStatistics \ + -XX:+UseParNewGC \ + -XX:+UseConcMarkSweepGC \ +-XX:+CMSParallelRemarkEnabled" +``` + +Salt init: + +```text +description "zookeeper-{{ cluster['name'] }} centralized coordination service" + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +limit nofile 8192 8192 + +pre-start script + [ -r "/etc/zookeeper-{{ cluster['name'] }}/conf/environment" ] || exit 0 + . /etc/zookeeper-{{ cluster['name'] }}/conf/environment + [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR + chown $USER:$GROUP $ZOO_LOG_DIR +end script + +script + . /etc/zookeeper-{{ cluster['name'] }}/conf/environment + [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper + if [ -z "$JMXDISABLE" ]; then + JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" + fi + exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ cluster['name'] }} \ + -- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \ + -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG +end script +``` + diff --git a/docs/en/operations/tips.rst b/docs/en/operations/tips.rst deleted file mode 100644 index 9390c7b2557..00000000000 --- a/docs/en/operations/tips.rst +++ /dev/null @@ -1,280 +0,0 @@ -Operations Tips -=============== - -CPU ---- - -SSE 4.2 instruction set support is required. Most recent (since 2008) CPUs have this instruction set. - -When choosing between CPU with more cores and slightly less frequency and CPU with less cores and more frequency, choose first. -For example, 16 cores with 2600 MHz is better than 8 cores with 3600 MHz. - -Hyper-Threading ---------------- - -Don't disable hyper-threading. Some queries will benefit from hyper-threading and some will not. - - -Turbo-Boost ------------ - -Don't disable turbo-boost. It will do significant performance gain on typical load. -Use ``turbostat`` tool to show real CPU frequency under load. - - -CPU scaling governor --------------------- - -Always use ``performance`` scaling governor. ``ondemand`` scaling governor performs much worse even on constantly high demand. - - .. code-block:: bash - - sudo echo 'performance' | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor - - -CPU throttling --------------- -Your CPU could be overheated. Use ``dmesg`` to show if it was thermal throttled. -Your CPU could be power capped in datacenter. Use ``turbostat`` tool under load to monitor that. - - -RAM ---- - -For small amount of data (up to ~200 GB compressed) prefer to use as much RAM as data volume. -For larger amount of data, if you run interactive (online) queries, use reasonable amount of RAM (128 GB or more) to hot data fit in page cache. -Even for data volumes of ~50 TB per server, using 128 GB of RAM is much better for query performance than 64 GB. - - -Swap ----- - -Disable swap. The only possible reason to not disable swap is when you are running ClickHouse on your personal laptop/desktop. - - -Huge pages ----------- - -Disable transparent huge pages. It interferes badly with memory allocators, leading to major performance degradation. - - .. code-block:: bash - - echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled - -Use ``perf top`` to monitor time spent in kernel on doing memory management. -Don't allocate permanent huge pages. - - -Storage subsystem ------------------ - -If you could afford SSD, use SSD. -Otherwise use HDD. SATA HDDs 7200 RPM are Ok. - -Prefer more servers with in place storage to less servers with huge disk shelves. -Of course you could use huge disk shelves for archive storage with rare queries. - - -RAID ----- - -When using HDDs, you could use RAID-10, RAID-5, RAID-6 or RAID-50. -Use Linux software RAID (``mdadm``). Better to not use LVM. -When creating RAID-10, choose ``far`` layout. -Prefer RAID-10 if you could afford it. - -Don't use RAID-5 on more than 4 HDDs - use RAID-6 or RAID-50. RAID-6 is better. -When using RAID-5, RAID-6, or RAID-50, always increase stripe_cache_size, because default setting is awful. - - .. code-block:: bash - - echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size - -Exact number is calculated from number of devices and chunk size: ``2 * num_devices * chunk_size_in_bytes / 4096``. - -Chunk size 1024K is Ok for all RAID configurations. -Never use too small or too large chunk sizes. - -On SSDs, you could use RAID-0. -Regardless to RAID, always use replication for data safety. - -Enable NCQ. Use high queue depth. Use CFQ scheduler for HDDs and noop for SSDs. Don't lower readahead setting. -Enable write cache on HDDs. - - -Filesystem ----------- - -Ext4 is Ok. Mount with ``noatime,nobarrier``. -XFS is Ok too, but less tested with ClickHouse. -Most other filesystems should work fine. Filesystems with delayed allocation are better. - - -Linux kernel ------------- - -Don't use too old Linux kernel. For example, on 2015, 3.18.19 is Ok. -You could use Yandex kernel: https://github.com/yandex/smart which gives at least 5% performance increase. - - -Network -------- - -When using IPv6, you must increase route cache. -Linux kernels before 3.2 has awful bugs in IPv6 implementation. - -Prefer at least 10 Gbit network. 1 Gbit will also work, but much worse for repairing replicas with tens of terabytes of data and for processing huge distributed queries with much intermediate data. - - -ZooKeeper ---------- - -Probably you already have ZooKeeper for other purposes. -It's Ok to use existing ZooKeeper installation if it is not overloaded. - -Use recent version of ZooKeeper. At least 3.4.9 is Ok. Version in your Linux package repository might be outdated. - -With default settings, ZooKeeper have time bomb: - - A ZooKeeper server will not remove old snapshots and log files when using the default configuration (see autopurge below), this is the responsibility of the operator. - -You need to defuse the bomb. - -Below is ZooKeeper (3.5.1) configuration used by Yandex.Metrica in production as of 2017-05-20. - -zoo.cfg: - -.. code-block:: bash - - # http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html - - # The number of milliseconds of each tick - tickTime=2000 - # The number of ticks that the initial - # synchronization phase can take - initLimit=30000 - # The number of ticks that can pass between - # sending a request and getting an acknowledgement - syncLimit=10 - - maxClientCnxns=2000 - - maxSessionTimeout=60000000 - # the directory where the snapshot is stored. - dataDir=/opt/zookeeper/{{ cluster['name'] }}/data - # Place the dataLogDir to a separate physical disc for better performance - dataLogDir=/opt/zookeeper/{{ cluster['name'] }}/logs - - autopurge.snapRetainCount=10 - autopurge.purgeInterval=1 - - - # To avoid seeks ZooKeeper allocates space in the transaction log file in - # blocks of preAllocSize kilobytes. The default block size is 64M. One reason - # for changing the size of the blocks is to reduce the block size if snapshots - # are taken more often. (Also, see snapCount). - preAllocSize=131072 - - # Clients can submit requests faster than ZooKeeper can process them, - # especially if there are a lot of clients. To prevent ZooKeeper from running - # out of memory due to queued requests, ZooKeeper will throttle clients so that - # there is no more than globalOutstandingLimit outstanding requests in the - # system. The default limit is 1,000.ZooKeeper logs transactions to a - # transaction log. After snapCount transactions are written to a log file a - # snapshot is started and a new transaction log file is started. The default - # snapCount is 10,000. - snapCount=3000000 - - # If this option is defined, requests will be will logged to a trace file named - # traceFile.year.month.day. - #traceFile= - - # Leader accepts client connections. Default value is "yes". The leader machine - # coordinates updates. For higher update throughput at thes slight expense of - # read throughput the leader can be configured to not accept clients and focus - # on coordination. - leaderServes=yes - - standaloneEnabled=false - dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic - ``` - -Java version: - -.. code-block:: text - - Java(TM) SE Runtime Environment (build 1.8.0_25-b17) - Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) - -JVM parameters: - -.. code-block:: bash - - NAME=zookeeper-{{ cluster['name'] }} - ZOOCFGDIR=/etc/$NAME/conf - - # TODO this is really ugly - # How to find out, which jars are needed? - # seems, that log4j requires the log4j.properties file to be in the classpath - CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper" - - ZOOCFG="$ZOOCFGDIR/zoo.cfg" - ZOO_LOG_DIR=/var/log/$NAME - USER=zookeeper - GROUP=zookeeper - PIDDIR=/var/run/$NAME - PIDFILE=$PIDDIR/$NAME.pid - SCRIPTNAME=/etc/init.d/$NAME - JAVA=/usr/bin/java - ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" - ZOO_LOG4J_PROP="INFO,ROLLINGFILE" - JMXLOCALONLY=false - JAVA_OPTS="-Xms{{ cluster.get('xms','128M') }} \ - -Xmx{{ cluster.get('xmx','1G') }} \ - -Xloggc:/var/log/$NAME/zookeeper-gc.log \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=16 \ - -XX:GCLogFileSize=16M \ - -verbose:gc \ - -XX:+PrintGCTimeStamps \ - -XX:+PrintGCDateStamps \ - -XX:+PrintGCDetails - -XX:+PrintTenuringDistribution \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+PrintGCApplicationConcurrentTime \ - -XX:+PrintSafepointStatistics \ - -XX:+UseParNewGC \ - -XX:+UseConcMarkSweepGC \ - -XX:+CMSParallelRemarkEnabled" - - -Инициализация через Salt: - -.. code-block:: text - - description "zookeeper-{{ cluster['name'] }} centralized coordination service" - - start on runlevel [2345] - stop on runlevel [!2345] - - respawn - - limit nofile 8192 8192 - - pre-start script - [ -r "/etc/zookeeper-{{ cluster['name'] }}/conf/environment" ] || exit 0 - . /etc/zookeeper-{{ cluster['name'] }}/conf/environment - [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR - chown $USER:$GROUP $ZOO_LOG_DIR - end script - - script - . /etc/zookeeper-{{ cluster['name'] }}/conf/environment - [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper - if [ -z "$JMXDISABLE" ]; then - JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" - fi - exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ cluster['name'] }} \ - -- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \ - -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG - end script diff --git a/docs/en/operators/index.md b/docs/en/operators/index.md new file mode 100644 index 00000000000..a8fe4c0ae15 --- /dev/null +++ b/docs/en/operators/index.md @@ -0,0 +1,122 @@ +# Operators + +All operators are transformed to the corresponding functions at the query parsing stage, in accordance with their precedence and associativity. +Groups of operators are listed in order of priority (the higher it is in the list, the earlier the operator is connected to its arguments). + +## Access operators + +`a[N]` Access to an element of an array; ` arrayElement(a, N) function`. + +`a.N` – Access to a tuble element; `tupleElement(a, N)` function. + +## Numeric negation operator + +`-a` – The `negate (a)` function. + +## Multiplication and division operators + +`a * b` – The `multiply (a, b) function.` + +`a / b` – The ` divide(a, b) function.` + +`a % b` – The `modulo(a, b) function.` + +## Addition and subtraction operators + +`a + b` – The `plus(a, b) function.` + +`a - b` – The `minus(a, b) function.` + +## Comparison operators + +`a = b` – The `equals(a, b) function.` + +`a == b` – The ` equals(a, b) function.` + +`a != b` – The `notEquals(a, b) function.` + +`a <> b` – The `notEquals(a, b) function.` + +`a <= b` – The `lessOrEquals(a, b) function.` + +`a >= b` – The `greaterOrEquals(a, b) function.` + +`a < b` – The `less(a, b) function.` + +`a > b` – The `greater(a, b) function.` + +`a LIKE s` – The `like(a, b) function.` + +`a NOT LIKE s` – The `notLike(a, b) function.` + +`a BETWEEN b AND c` – The same as `a >= b AND a <= c.` + +## Operators for working with data sets + +*See the section "IN operators".* + +`a IN ...` – The `in(a, b) function` + +`a NOT IN ...` – The `notIn(a, b) function.` + +`a GLOBAL IN ...` – The `globalIn(a, b) function.` + +`a GLOBAL NOT IN ...` – The `globalNotIn(a, b) function.` + +## Logical negation operator + +`NOT a` The `not(a) function.` + +## Logical "AND" operator. + +`a AND b` – The`and(a, b) function.` + +## Logical "OR" operator. + +`a OR b` – The `or(a, b) function.` + +## Conditional operator + +`a ? b : c` – The `if(a, b, c) function.` + +Note: + +The conditional operator calculates the values of b and c, then checks whether condition a is met, and then returns the corresponding value. If "b" or "c" is an arrayJoin() function, each row will be replicated regardless of the "a" condition. + +## Conditional expression + +```sql +CASE [x] + WHEN a THEN b + [WHEN ... THEN ...] + ELSE c +END +``` + +If "x" is specified, then transform(x, \[a, ...\], \[b, ...\], c). Otherwise – multiIf(a, b, ..., c). + +## Concatenation operator + +`s1 || s2` – The `concat(s1, s2) function.` + +## Lambda creation operator + +`x -> expr` – The `lambda(x, expr) function.` + +The following operators do not have a priority, since they are brackets: + +## Array creation operator + +`[x1, ...]` – The `array(x1, ...) function.` + +## Tuple creation operator + +`(x1, x2, ...)` – The `tuple(x2, x2, ...) function.` + +## Associativity + +All binary operators have left associativity. For example, `1 + 2 + 3` is transformed to `plus(plus(1, 2), 3)`. +Sometimes this doesn't work the way you expect. For example, ` SELECT 4 > 2 > 3` will result in 0. + +For efficiency, the `and` and `or` functions accept any number of arguments. The corresponding chains of `AND` and `OR` operators are transformed to a single call of these functions. + diff --git a/docs/en/operators/index.rst b/docs/en/operators/index.rst deleted file mode 100644 index 999c701217f..00000000000 --- a/docs/en/operators/index.rst +++ /dev/null @@ -1,138 +0,0 @@ -Operators -========= - -All operators are transformed to the corresponding functions at the query parsing stage, in accordance with their precedence and associativity. - -Access operators ----------------- - -``a[N]`` - Access to an array element, arrayElement(a, N) function. - -``a.N`` - Access to a tuple element, tupleElement(a, N) function. - -Numeric negation operator -------------------------- - -``-a`` - negate(a) function - -Multiplication and division operators -------------------------------------- - -``a * b`` - multiply(a, b) function - -``a / b`` - divide(a, b) function - -``a % b`` - modulo(a, b) function - -Addition and subtraction operators ----------------------------------- - -``a + b`` - plus(a, b) function - -``a - b`` - minus(a, b) function - -Comparison operators --------------------- - -``a = b`` - equals(a, b) function - -``a == b`` - equals(a, b) function - -``a != b`` - notEquals(a, b) function - -``a <> b`` - notEquals(a, b) function - -``a <= b`` - lessOrEquals(a, b) function - -``a >= b`` - greaterOrEquals(a, b) function - -``a < b`` - less(a, b) function - -``a > b`` - greater(a, b) function - -``a LIKE s`` - like(a, b) function - -``a NOT LIKE s`` - notLike(a, b) function - -``a BETWEEN b AND c`` - equivalent to a >= b AND a <= c - - -Operators for working with data sets ------------------------------------- - -*See the section "IN operators".* - -``a IN ...`` - in(a, b) function - -``a NOT IN ...`` - notIn(a, b) function - -``a GLOBAL IN ...`` - globalIn(a, b) function - -``a GLOBAL NOT IN ...`` - globalNotIn(a, b) function - - - -Logical negation operator -------------------------- - -``NOT a`` - ``not(a)`` function - - -Logical "AND" operator ----------------------- - -``a AND b`` - function ``and(a, b)`` - - -Logical "OR" operator ---------------------- - -``a OR b`` - function ``or(a, b)`` - -Conditional operator --------------------- - -``a ? b : c`` - function ``if(a, b, c)`` - -Conditional expression ----------------------- - -.. code-block:: sql - - CASE [x] - WHEN a THEN b - [WHEN ... THEN ...] - ELSE c - END - -If x is given - transform(x, [a, ...], [b, ...], c). Otherwise, multiIf(a, b, ..., c). - -String concatenation operator ------------------------------ - -``s1 || s2`` - concat(s1, s2) function - -Lambda creation operator ------------------------- - -``x -> expr`` - lambda(x, expr) function - -The following operators do not have a priority, since they are brackets: - -Array creation operator ------------------------ - -``[x1, ...]`` - array(x1, ...) function - -Tuple creation operator ------------------------ -``(x1, x2, ...)`` - tuple(x2, x2, ...) function - - -Associativity -------------- - -All binary operators have left associativity. For example, ``'1 + 2 + 3'`` is transformed to ``'plus(plus(1, 2), 3)'``. -Sometimes this doesn't work the way you expect. For example, ``'SELECT 4 > 3 > 2'`` results in ``0``. - -For efficiency, the 'and' and 'or' functions accept any number of arguments. The corresponding chains of AND and OR operators are transformed to a single call of these functions. diff --git a/docs/en/query_language/clickhouse_local.md b/docs/en/query_language/clickhouse_local.md new file mode 100644 index 00000000000..d18cc200320 --- /dev/null +++ b/docs/en/query_language/clickhouse_local.md @@ -0,0 +1,4 @@ +# The clickhouse-local program + +The `clickhouse-local` program enables you to perform fast processing on local files that store tables, without having to deploy and configure clickhouse-server. + diff --git a/docs/en/query_language/clickhouse_local.rst b/docs/en/query_language/clickhouse_local.rst deleted file mode 100644 index 13b5285d6a0..00000000000 --- a/docs/en/query_language/clickhouse_local.rst +++ /dev/null @@ -1,4 +0,0 @@ -clickhouse-local ----------------- - -Application ``clickhouse-local`` can fast processing of local files that store tables without resorting to deployment and configuration clickhouse-server ... diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md new file mode 100644 index 00000000000..27acbf61e50 --- /dev/null +++ b/docs/en/query_language/index.md @@ -0,0 +1,9 @@ +# Query language + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/query_language/index.rst b/docs/en/query_language/index.rst deleted file mode 100644 index 64e399de525..00000000000 --- a/docs/en/query_language/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Query language -============== - -.. toctree:: - :glob: - - * diff --git a/docs/en/query_language/queries.md b/docs/en/query_language/queries.md new file mode 100644 index 00000000000..aa506ba9e2c --- /dev/null +++ b/docs/en/query_language/queries.md @@ -0,0 +1,1508 @@ +#Queries + +## CREATE DATABASE + +Creating db_name databases + +```sql +CREATE DATABASE [IF NOT EXISTS] db_name +``` + +`A database` is just a directory for tables. +If `IF NOT EXISTS` is included, the query won't return an error if the database already exists. + +## CREATE TABLE + +The `CREATE TABLE` query can have several forms. + +```sql +CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = engine +``` + +Creates a table named 'name' in the 'db' database or the current database if 'db' is not set, with the structure specified in brackets and the 'engine' engine. +The structure of the table is a list of column descriptions. If indexes are supported by the engine, they are indicated as parameters for the table engine. + +A column description is `name type` in the simplest case. Example: `RegionID UInt32`. +Expressions can also be defined for default values (see below). + +```sql +CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name AS [db2.]name2 [ENGINE = engine] +``` + +Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. + +```sql +CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name ENGINE = engine AS SELECT ... +``` + +Creates a table with a structure like the result of the `SELECT` query, with the 'engine' engine, and fills it with data from SELECT. + +In all cases, if `IF NOT EXISTS` is specified, the query won't return an error if the table already exists. In this case, the query won't do anything. + +### Default values + +The column description can specify an expression for a default value, in one of the following ways:`DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. +Example: `URLDomain String DEFAULT domain(URL)`. + +If an expression for the default value is not defined, the default values will be set to zeros for numbers, empty strings for strings, empty arrays for arrays, and `0000-00-00` for dates or `0000-00-00 00:00:00` for dates with time. NULLs are not supported. + +If the default expression is defined, the column type is optional. If there isn't an explicitly defined type, the default expression type is used. Example: `EventDate DEFAULT toDate(EventTime)` – the 'Date' type will be used for the 'EventDate' column. + +If the data type and default expression are defined explicitly, this expression will be cast to the specified type using type casting functions. Example: `Hits UInt32 DEFAULT 0` means the same thing as `Hits UInt32 DEFAULT toUInt32(0)`. + +Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don't contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. + +`DEFAULT expr` + +Normal default value. If the INSERT query doesn't specify the corresponding column, it will be filled in by computing the corresponding expression. + +`MATERIALIZED expr` + +Materialized expression. Such a column can't be specified for INSERT, because it is always calculated. +For an INSERT without a list of columns, these columns are not considered. +In addition, this column is not substituted when using an asterisk in a SELECT query. This is to preserve the invariant that the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. + +`ALIAS expr` + +Synonym. Such a column isn't stored in the table at all. +Its values can't be inserted in a table, and it is not substituted when using an asterisk in a SELECT query. +It can be used in SELECTs if the alias is expanded during query parsing. + +When using the ALTER query to add new columns, old data for these columns is not written. Instead, when reading old data that does not have values for the new columns, expressions are computed on the fly by default. However, if running the expressions requires different columns that are not indicated in the query, these columns will additionally be read, but only for the blocks of data that need it. + +If you add a new column to a table but later change its default expression, the values used for old data will change (for data where values were not stored on the disk). Note that when running background merges, data for columns that are missing in one of the merging parts is written to the merged part. + +It is not possible to set default values for elements in nested data structures. + +### Temporary tables + +In all cases, if `TEMPORARY` is specified, a temporary table will be created. Temporary tables have the following characteristics: + +- Temporary tables disappear when the session ends, including if the connection is lost. +- A temporary table is created with the Memory engine. The other table engines are not supported. +- The DB can't be specified for a temporary table. It is created outside of databases. +- If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used. +- For distributed query processing, temporary tables used in a query are passed to remote servers. + +In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed `(GLOBAL) IN`. For more information, see the appropriate sections + +Distributed DDL queries (ON CLUSTER clause) +---------------------------------------------- + +The `CREATE`, `DROP`, `ALTER`, and `RENAME` queries support distributed execution on a cluster. +For example, the following query creates the `all_hits` `Distributed` table on each host in `cluster`: + +```sql +CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, hits) +``` + +In order to run these queries correctly, each host must have the same cluster definition (to simplify syncing configs, you can use substitutions from ZooKeeper). They must also connect to the ZooKeeper servers. +The local version of the query will eventually be implemented on each host in the cluster, even if some hosts are currently not available. The order for executing queries within a single host is guaranteed. +` ALTER` queries are not yet supported for replicated tables. + +## CREATE VIEW + +```sql +CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]name [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... +``` + +Creates a view. There are two types of views: normal and MATERIALIZED. + +When creating a materialized view, you must specify ENGINE – the table engine for storing data. + +A materialized view works as follows: when inserting data to the table specified in SELECT, part of the inserted data is converted by this SELECT query, and the result is inserted in the view. + +Normal views don't store any data, but just perform a read from another table. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the FROM clause. + +As an example, assume you've created a view: + +```sql +CREATE VIEW view AS SELECT ... +``` + +and written a query: + +```sql +SELECT a, b, c FROM view +``` + +This query is fully equivalent to using the subquery: + +```sql +SELECT a, b, c FROM (SELECT ...) +``` + +Materialized views store data transformed by the corresponding SELECT query. + +When creating a materialized view, you must specify ENGINE – the table engine for storing data. + +A materialized view is arranged as follows: when inserting data to the table specified in SELECT, part of the inserted data is converted by this SELECT query, and the result is inserted in the view. + +If you specify POPULATE, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We don't recommend using POPULATE, since data inserted in the table during the view creation will not be inserted in it. + +A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`... Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won't be further aggregated. The exception is when using an ENGINE that independently performs data aggregation, such as `SummingMergeTree`. + +The execution of `ALTER` queries on materialized views has not been fully developed, so they might be inconvenient. If the materialized view uses the construction ``TO [db.]name``, you can ``DETACH`` the view, run ``ALTER`` for the target table, and then ``ATTACH`` the previously detached (``DETACH``) view. + +Views look the same as normal tables. For example, they are listed in the result of the `SHOW TABLES` query. + +There isn't a separate query for deleting views. To delete a view, use `DROP TABLE`. + +## ATTACH + +This query is exactly the same as `CREATE`, but + +- instead of the word `CREATE` it uses the word `ATTACH`. +- The query doesn't create data on the disk, but assumes that data is already in the appropriate places, and just adds information about the table to the server. +After executing an ATTACH query, the server will know about the existence of the table. + +If the table was previously detached (``DETACH``), meaning that its structure is known, you can use shorthand without defining the structure. + +```sql +ATTACH TABLE [IF NOT EXISTS] [db.]name +``` + +This query is used when starting the server. The server stores table metadata as files with `ATTACH` queries, which it simply runs at launch (with the exception of system tables, which are explicitly created on the server). + +## DROP + +This query has two types: `DROP DATABASE` and `DROP TABLE`. + +```sql +DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] +``` + +Deletes all tables inside the 'db' database, then deletes the 'db' database itself. +If `IF EXISTS` is specified, it doesn't return an error if the database doesn't exist. + +```sql +DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +``` + +Deletes the table. +If `IF EXISTS` is specified, it doesn't return an error if the table doesn't exist or the database doesn't exist. + +## DETACH + +Deletes information about the 'name' table from the server. The server stops knowing about the table's existence. + +```sql +DETACH TABLE [IF EXISTS] [db.]name +``` + +This does not delete the table's data or metadata. On the next server launch, the server will read the metadata and find out about the table again. +Similarly, a "detached" table can be re-attached using the `ATTACH` query (with the exception of system tables, which do not have metadata stored for them). + +There is no `DETACH DATABASE` query. + +## RENAME + +Renames one or more tables. + +```sql +RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... [ON CLUSTER cluster] +``` + +All tables are renamed under global locking. Renaming tables is a light operation. If you indicated another database after TO, the table will be moved to this database. However, the directories with databases must reside in the same file system (otherwise, an error is returned). + + + +## ALTER + +The `ALTER` query is only supported for `*MergeTree` tables, as well as `Merge`and`Distributed`. The query has several variations. + +### Column manipulations + +Changing the table structure. + +```sql +ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|MODIFY COLUMN ... +``` + +In the query, specify a list of one or more comma-separated actions. +Each action is an operation on a column. + +The following actions are supported: + +```sql +ADD COLUMN name [type] [default_expr] [AFTER name_after] +``` + +Adds a new column to the table with the specified name, type, and `default_expr` (see the section "Default expressions"). If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, 'name_after' can be the name of a column that is added in one of the previous actions. + +Adding a column just changes the table structure, without performing any actions with data. The data doesn't appear on the disk after ALTER. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see MergeTree). + +This approach allows us to complete the ALTER query instantly, without increasing the volume of old data. + +```sql +DROP COLUMN name +``` + +Deletes the column with the name 'name'. +Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly. + +```sql +MODIFY COLUMN name [type] [default_expr] +``` + +Changes the 'name' column's type to 'type' and/or the default expression to 'default_expr'. When changing the type, values are converted as if the 'toType' function were applied to them. + +If only the default expression is changed, the query doesn't do anything complex, and is completed almost instantly. + +Changing the column type is the only complex action – it changes the contents of files with data. For large tables, this may take a long time. + +There are several processing stages: + +- Preparing temporary (new) files with modified data. +- Renaming old files. +- Renaming the temporary (new) files to the old names. +- Deleting the old files. + +Only the first stage takes time. If there is a failure at this stage, the data is not changed. +If there is a failure during one of the successive stages, data can be restored manually. The exception is if the old files were deleted from the file system but the data for the new files did not get written to the disk and was lost. + +There is no support for changing the column type in arrays and nested data structures. + +The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot. + +There is no support for deleting columns in the primary key or the sampling key (columns that are in the `ENGINE` expression). Changing the type for columns that are included in the primary key is only possible if this change does not cause the data to be modified (for example, it is allowed to add values to an Enum or change a type with `DateTime` to `UInt32`). + +If the `ALTER` query is not sufficient for making the table changes you need, you can create a new table, copy the data to it using the `INSERT SELECT` query, then switch the tables using the `RENAME` query and delete the old table. + +The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running. + +For tables that don't store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers. + +The `ALTER` query for changing columns is replicated. The instructions are saved in ZooKeeper, then each replica applies them. All `ALTER` queries are run in the same order. The query waits for the appropriate actions to be completed on the other replicas. However, a query to change columns in a replicated table can be interrupted, and all actions will be performed asynchronously. + +### Manipulations with partitions and parts + +It only works for tables in the `MergeTree` family. The following operations are available: + +- `DETACH PARTITION` – Move a partition to the 'detached' directory and forget it. +- `DROP PARTITION` – Delete a partition. +- `ATTACH PART|PARTITION` – Add a new part or partition from the `detached` directory to the table. +- `FREEZE PARTITION` – Create a backup of a partition. +- `FETCH PARTITION` – Download a partition from another server. + +Each type of query is covered separately below. + +A partition in a table is data for a single calendar month. This is determined by the values of the date key specified in the table engine parameters. Each month's data is stored separately in order to simplify manipulations with this data. + +A "part" in the table is part of the data from a single partition, sorted by the primary key. + +You can use the `system.parts` table to view the set of table parts and partitions: + +```sql +SELECT * FROM system.parts WHERE active +``` + +`active` – Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part – these parts are deleted approximately 10 minutes after merging. + +Another way to view a set of parts and partitions is to go into the directory with table data. +Data directory: `/var/lib/clickhouse/data/database/table/`, +where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: + +```bash +$ ls -l /var/lib/clickhouse/data/test/visits/ +total 48 +drwxrwxrwx 2 clickhouse clickhouse 20480 may 13 02:58 20140317_20140323_2_2_0 +drwxrwxrwx 2 clickhouse clickhouse 20480 may 13 02:58 20140317_20140323_4_4_0 +drwxrwxrwx 2 clickhouse clickhouse 4096 may 13 02:55 detached +-rw-rw-rw- 1 clickhouse clickhouse 2 may 13 02:58 increment.txt +``` + +Here, `20140317_20140323_2_2_0` and ` 20140317_20140323_4_4_0` are the directories of data parts. + +Let's break down the name of the first part: `20140317_20140323_2_2_0`. + +- `20140317` is the minimum date of the data in the chunk. +- `20140323` is the maximum data of the data in the chunk. +- `2` is the minimum number of the data block. +- `2` is the maximum number of the data block. +- `0` is the chunk level (the depth of the merge tree it is formed from). + +Each piece relates to a single partition and contains data for just one month. +`201403` is the name of the partition. A partition is a set of parts for a single month. + +On an operating server, you can't manually change the set of parts or their data on the file system, since the server won't know about it. +For non-replicated tables, you can do this when the server is stopped, but we don't recommended it. +For replicated tables, the set of parts can't be changed in any case. + +The `detached` directory contains parts that are not used by the server - detached from the table using the `ALTER ... DETACH` query. Parts that are damaged are also moved to this directory, instead of deleting them. You can add, delete, or modify the data in the 'detached' directory at any time – the server won't know about this until you make the `ALTER TABLE ... ATTACH` query. + +```sql +ALTER TABLE [db.]table DETACH PARTITION 'name' +``` + +Move all data for partitions named 'name' to the 'detached' directory and forget about them. +The partition name is specified in YYYYMM format. It can be indicated in single quotes or without them. + +After the query is executed, you can do whatever you want with the data in the 'detached' directory — delete it from the file system, or just leave it. + +The query is replicated – data will be moved to the 'detached' directory and forgotten on all replicas. The query can only be sent to a leader replica. To find out if a replica is a leader, perform SELECT to the 'system.replicas' system table. Alternatively, it is easier to make a query on all replicas, and all except one will throw an exception. + +```sql +ALTER TABLE [db.]table DROP PARTITION 'name' +``` + +The same as the `DETACH` operation. Deletes data from the table. Data parts will be tagged as inactive and will be completely deleted in approximately 10 minutes. The query is replicated – data will be deleted on all replicas. + +```sql +ALTER TABLE [db.]table ATTACH PARTITION|PART 'name' +``` + +Adds data to the table from the 'detached' directory. + +It is possible to add data for an entire partition or a separate part. For a part, specify the full name of the part in single quotes. + +The query is replicated. Each replica checks whether there is data in the 'detached' directory. If there is data, it checks the integrity, verifies that it matches the data on the server that initiated the query, and then adds it if everything is correct. If not, it downloads data from the query requestor replica, or from another replica where the data has already been added. + +So you can put data in the 'detached' directory on one replica, and use the ALTER ... ATTACH query to add it to the table on all replicas. + +```sql +ALTER TABLE [db.]table FREEZE PARTITION 'name' +``` + +Creates a local backup of one or multiple partitions. The name can be the full name of the partition (for example, 201403), or its prefix (for example, 2014): then the backup will be created for all the corresponding partitions. + +The query does the following: for a data snapshot at the time of execution, it creates hardlinks to table data in the directory `/var/lib/clickhouse/shadow/N/...` + +`/var/lib/clickhouse/` is the working ClickHouse directory from the config. +`N` is the incremental number of the backup. + +The same structure of directories is created inside the backup as inside `/var/lib/clickhouse/`. +It also performs 'chmod' for all files, forbidding writes to them. + +The backup is created almost instantly (but first it waits for current queries to the corresponding table to finish running). At first, the backup doesn't take any space on the disk. As the system works, the backup can take disk space, as data is modified. If the backup is made for old enough data, it won't take space on the disk. + +After creating the backup, data from `/var/lib/clickhouse/shadow/` can be copied to the remote server and then deleted on the local server. +The entire backup process is performed without stopping the server. + +The `ALTER ... FREEZE PARTITION` query is not replicated. A local backup is only created on the local server. + +As an alternative, you can manually copy data from the `/var/lib/clickhouse/data/database/table` directory. +But if you do this while the server is running, race conditions are possible when copying directories with files being added or changed, and the backup may be inconsistent. You can do this if the server isn't running – then the resulting data will be the same as after the `ALTER TABLE t FREEZE PARTITION` query. + +`ALTER TABLE ... FREEZE PARTITION` only copies data, not table metadata. To make a backup of table metadata, copy the file `/var/lib/clickhouse/metadata/database/table.sql` + +To restore from a backup: + +> - Use the CREATE query to create the table if it doesn't exist. The query can be taken from an .sql file (replace `ATTACH` in it with `CREATE`). +- Copy the data from the data/database/table/ directory inside the backup to the `/var/lib/clickhouse/data/database/table/detached/ directory.` +- Run `ALTER TABLE ... ATTACH PARTITION YYYYMM` queries, where `YYYYMM` is the month, for every month. + +In this way, data from the backup will be added to the table. +Restoring from a backup doesn't require stopping the server. + +### Backups and replication + +Replication provides protection from device failures. If all data disappeared on one of your replicas, follow the instructions in the "Restoration after failure" section to restore it. + +For protection from device failures, you must use replication. For more information about replication, see the section "Data replication". + +Backups protect against human error (accidentally deleting data, deleting the wrong data or in the wrong cluster, or corrupting data). +For high-volume databases, it can be difficult to copy backups to remote servers. In such cases, to protect from human error, you can keep a backup on the same server (it will reside in `/var/lib/clickhouse/shadow/`). + +```sql +ALTER TABLE [db.]table FETCH PARTITION 'name' FROM 'path-in-zookeeper' +``` + +This query only works for replicatable tables. + +It downloads the specified partition from the shard that has its `ZooKeeper path` specified in the `FROM` clause, then puts it in the `detached` directory for the specified table. + +Although the query is called `ALTER TABLE`, it does not change the table structure, and does not immediately change the data available in the table. + +Data is placed in the `detached` directory. You can use the `ALTER TABLE ... ATTACH` query to attach the data. + +The ` FROM` clause specifies the path in ` ZooKeeper`. For example, `/clickhouse/tables/01-01/visits`. +Before downloading, the system checks that the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. + +The `ALTER ... FETCH PARTITION` query is not replicated. The partition will be downloaded to the 'detached' directory only on the local server. Note that if after this you use the `ALTER TABLE ... ATTACH` query to add data to the table, the data will be added on all replicas (on one of the replicas it will be added from the 'detached' directory, and on the rest it will be loaded from neighboring replicas). + +### Synchronicity of ALTER queries + +For non-replicatable tables, all `ALTER` queries are performed synchronously. For replicatable tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. + +For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_partitions_sync` setting to set up waiting. +Possible values: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all. + + + +## SHOW DATABASES + +```sql +SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] +``` + +Prints a list of all databases. +This query is identical to `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. + +See also the section "Formats". + +## SHOW TABLES + +```sql +SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] +``` + +Displays a list of tables + +- tables from the current database, or from the 'db' database if "FROM db" is specified. +- all tables, or tables whose name matches the pattern, if "LIKE 'pattern'" is specified. + +This query is identical to: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]` +Смотрите также раздел "Оператор LIKE". + +## SHOW PROCESSLIST + +```sql +SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] +``` + +Outputs a list of queries currently being processed, other than `SHOW PROCESSLIST` queries. + +Prints a table containing the columns: + +**user** – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the 'default' user. SHOW PROCESSLIST shows the username for a specific query, not for a query that this query initiated. + +**address** – The name of the host that the query was sent from. For distributed processing, on remote servers, this is the name of the query requestor host. To track where a distributed query was originally made from, look at SHOW PROCESSLIST on the query requestor server. + +**elapsed** – The execution time, in seconds. Queries are output in order of decreasing execution time. + +**rows_read**, **bytes_read** – How many rows and bytes of uncompressed data were read when processing the query. For distributed processing, data is totaled from all the remote servers. This is the data used for restrictions and quotas. + +**memory_usage** – Current RAM usage in bytes. See the setting 'max_memory_usage'. + +**query** – The query itself. In INSERT queries, the data for insertion is not output. + +**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. + +This query is identical to: `SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]`. + +Tip (execute in the console): + +```bash +watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" +``` + +## SHOW CREATE TABLE + +```sql +SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +``` + +Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table. + +## DESCRIBE TABLE + +```sql +DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +``` + +Returns two `String`-type columns: `name` and `type`, which indicate the names and types of columns in the specified table. + +Nested data structures are output in "expanded" format. Each column is shown separately, with the name after a dot. + +## EXISTS + +```sql +EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] +``` + +Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database. + +## USE + +```sql +USE db +``` + +Lets you set the current database for the session. +The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name. +This query can't be made when using the HTTP protocol, since there is no concept of a session. + +## SET + +```sql +SET param = value +``` + +Allows you to set `param` to `value`. You can also make all the settings from the specified settings profile in a single query. To do this, specify 'profile' as the setting name. For more information, see the section "Settings". +The setting is made for the session, or for the server (globally) if `GLOBAL` is specified. +When making a global setting, the setting is not applied to sessions already running, including the current session. It will only be used for new sessions. + +When the server is restarted, global settings made using `SET` are lost. +To make settings that persist after a server restart, you can only use the server's config file. + +## OPTIMIZE + +```sql +OPTIMIZE TABLE [db.]name [PARTITION partition] [FINAL] +``` + +Asks the table engine to do something for optimization. +Supported only by `*MergeTree` engines, in which this query initializes a non-scheduled merge of data parts. +If you specify a `PARTITION`, only the specified partition will be optimized. +If you specify `FINAL`, optimization will be performed even when all the data is already in one part. + + + +## INSERT + +Adding data. + +Basic query format: + +```sql +INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... +``` + +The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case, the rest of the columns are filled with: + +- The values calculated from the `DEFAULT` expressions specified in the table definition. +- Zeros and empty strings, if `DEFAULT` expressions are not defined. + +If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have ` DEFAULT` defined must be listed in the query. + +The INSERT can pass data in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: + +```sql +INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set +``` + +For example, the following query format is identical to the basic version of INSERT ... VALUES: + +```sql +INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... +``` + +ClickHouse removes all spaces and one line feed (if there is one) before the data. When forming a query, we recommend putting the data on a new line after the query operators (this is important if the data begins with spaces). + +Example: + +```sql +INSERT INTO t FORMAT TabSeparated +11 Hello, world! +22 Qwerty +``` + +You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section "[Interfaces](../interfaces/index.md#interfaces)". + +### Inserting the results of `SELECT` + +```sql +INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... +``` + +Columns are mapped according to their position in the SELECT clause. However, their names in the SELECT expression and the table for INSERT may differ. If necessary, type casting is performed. + +None of the data formats except Values allow setting values to expressions such as `now()`, `1 + 2`, and so on. The Values format allows limited use of expressions, but this is not recommended, because in this case inefficient code is used for their execution. + +Other queries for modifying data parts are not supported: `UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`. +However, you can delete old data using `ALTER TABLE ... DROP PARTITION`. + +### Performance considerations + +`INSERT` sorts the input data by primary key and splits them into partitions by month. If you insert data for mixed months, it can significantly reduce the performance of the `INSERT` query. To avoid this: + +- Add data in fairly large batches, such as 100,000 rows at a time. +- Group data by month before uploading it to ClickHouse. + +Performance will not decrease if: + +- Data is added in real time. +- You upload data that is usually sorted by time. + +## SELECT + +Data sampling. + +```sql +SELECT [DISTINCT] expr_list + [FROM [db.]table | (subquery) | table_function] [FINAL] + [SAMPLE sample_coeff] + [ARRAY JOIN ...] + [GLOBAL] ANY|ALL INNER|LEFT JOIN (subquery)|table USING columns_list + [PREWHERE expr] + [WHERE expr] + [GROUP BY expr_list] [WITH TOTALS] + [HAVING expr] + [ORDER BY expr_list] + [LIMIT [n, ]m] + [UNION ALL ...] + [INTO OUTFILE filename] + [FORMAT format] + [LIMIT n BY columns] +``` + +All the clauses are optional, except for the required list of expressions immediately after SELECT. +The clauses below are described in almost the same order as in the query execution conveyor. + +If the query omits the `DISTINCT`, `GROUP BY` and `ORDER BY` clauses and the `IN` and `JOIN` subqueries, the query will be completely stream processed, using O(1) amount of RAM. +Otherwise, the query might consume a lot of RAM if the appropriate restrictions are not specified: `max_memory_usage`, `max_rows_to_group_by`, `max_rows_to_sort`, `max_rows_in_distinct`, `max_bytes_in_distinct`, `max_rows_in_set`, `max_bytes_in_set`, `max_rows_in_join`, `max_bytes_in_join`, `max_bytes_before_external_sort`, `max_bytes_before_external_group_by`. For more information, see the section "Settings". It is possible to use external sorting (saving temporary tables to a disk) and external aggregation. `The system does not have "merge join"`. + +### FROM clause + +If the FROM clause is omitted, data will be read from the `system.one` table. +The 'system.one' table contains exactly one row (this table fulfills the same purpose as the DUAL table found in other DBMSs). + +The FROM clause specifies the table to read data from, or a subquery, or a table function; ARRAY JOIN and the regular JOIN may also be included (see below). + +Instead of a table, the SELECT subquery may be specified in brackets. +In this case, the subquery processing pipeline will be built into the processing pipeline of an external query. +In contrast to standard SQL, a synonym does not need to be specified after a subquery. For compatibility, it is possible to write 'AS name' after a subquery, but the specified name isn't used anywhere. + +A table function may be specified instead of a table. For more information, see the section "Table functions". + +To execute a query, all the columns listed in the query are extracted from the appropriate table. Any columns not needed for the external query are thrown out of the subqueries. +If a query does not list any columns (for example, SELECT count() FROM t), some column is extracted from the table anyway (the smallest one is preferred), in order to calculate the number of rows. + +The FINAL modifier can be used only for a SELECT from a CollapsingMergeTree table. When you specify FINAL, data is selected fully "collapsed". Keep in mind that using FINAL leads to a selection that includes columns related to the primary key, in addition to the columns specified in the SELECT. Additionally, the query will be executed in a single stream, and data will be merged during query execution. This means that when using FINAL, the query is processed more slowly. In most cases, you should avoid using FINAL. For more information, see the section "CollapsingMergeTree engine". + +### SAMPLE clause + +The SAMPLE clause allows for approximated query processing. Approximated query processing is only supported by MergeTree\* type tables, and only if the sampling expression was specified during table creation (see the section "MergeTree engine"). + +`SAMPLE` has the `format SAMPLE k`, where `k` is a decimal number from 0 to 1, or `SAMPLE n`, where 'n' is a sufficiently large integer. + +In the first case, the query will be executed on 'k' percent of data. For example, `SAMPLE 0.1` runs the query on 10% of data. +In the second case, the query will be executed on a sample of no more than 'n' rows. For example, `SAMPLE 10000000` runs the query on a maximum of 10,000,000 rows. + +Example: + +```sql +SELECT + Title, + count() * 10 AS PageViews +FROM hits_distributed +SAMPLE 0.1 +WHERE + CounterID = 34 + AND toDate(EventDate) >= toDate('2013-01-29') + AND toDate(EventDate) <= toDate('2013-02-04') + AND NOT DontCountHits + AND NOT Refresh + AND Title != '' +GROUP BY Title +ORDER BY PageViews DESC LIMIT 1000 +``` + +In this example, the query is executed on a sample from 0.1 (10%) of data. Values of aggregate functions are not corrected automatically, so to get an approximate result, the value 'count()' is manually multiplied by 10. + +When using something like `SAMPLE 10000000`, there isn't any information about which relative percent of data was processed or what the aggregate functions should be multiplied by, so this method of writing is not always appropriate to the situation. + +A sample with a relative coefficient is "consistent": if we look at all possible data that could be in the table, a sample (when using a single sampling expression specified during table creation) with the same coefficient always selects the same subset of possible data. In other words, a sample from different tables on different servers at different times is made the same way. + +For example, a sample of user IDs takes rows with the same subset of all the possible user IDs from different tables. This allows using the sample in subqueries in the IN clause, as well as for manually correlating results of different queries with samples. + +### ARRAY JOIN clause + +Allows executing JOIN with an array or nested data structure. The intent is similar to the 'arrayJoin' function, but its functionality is broader. + +`ARRAY JOIN` is essentially `INNER JOIN` with an array. Example: + +```text +:) CREATE TABLE arrays_test (s String, arr Array(UInt8)) ENGINE = Memory + +CREATE TABLE arrays_test +( + s String, + arr Array(UInt8) +) ENGINE = Memory + +Ok. + +0 rows in set. Elapsed: 0.001 sec. + +:) INSERT INTO arrays_test VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []) + +INSERT INTO arrays_test VALUES + +Ok. + +3 rows in set. Elapsed: 0.001 sec. + +:) SELECT * FROM arrays_test + +SELECT * +FROM arrays_test + +┌─s───────┬─arr─────┐ +│ Hello │ [1,2] │ +│ World │ [3,4,5] │ +│ Goodbye │ [] │ +└─────────┴─────────┘ + +3 rows in set. Elapsed: 0.001 sec. + +:) SELECT s, arr FROM arrays_test ARRAY JOIN arr + +SELECT s, arr +FROM arrays_test +ARRAY JOIN arr + +┌─s─────┬─arr─┐ +│ Hello │ 1 │ +│ Hello │ 2 │ +│ World │ 3 │ +│ World │ 4 │ +│ World │ 5 │ +└───────┴─────┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +An alias can be specified for an array in the ARRAY JOIN clause. In this case, an array item can be accessed by this alias, but the array itself by the original name. Example: + +```text +:) SELECT s, arr, a FROM arrays_test ARRAY JOIN arr AS a + +SELECT s, arr, a +FROM arrays_test +ARRAY JOIN arr AS a + +┌─s─────┬─arr─────┬─a─┐ +│ Hello │ [1,2] │ 1 │ +│ Hello │ [1,2] │ 2 │ +│ World │ [3,4,5] │ 3 │ +│ World │ [3,4,5] │ 4 │ +│ World │ [3,4,5] │ 5 │ +└───────┴─────────┴───┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +Multiple arrays of the same size can be comma-separated in the ARRAY JOIN clause. In this case, JOIN is performed with them simultaneously (the direct sum, not the direct product). Example: + +```text +:) SELECT s, arr, a, num, mapped FROM arrays_test ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped + +SELECT s, arr, a, num, mapped +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(lambda(tuple(x), plus(x, 1)), arr) AS mapped + +┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ +│ Hello │ [1,2] │ 1 │ 1 │ 2 │ +│ Hello │ [1,2] │ 2 │ 2 │ 3 │ +│ World │ [3,4,5] │ 3 │ 1 │ 4 │ +│ World │ [3,4,5] │ 4 │ 2 │ 5 │ +│ World │ [3,4,5] │ 5 │ 3 │ 6 │ +└───────┴─────────┴───┴─────┴────────┘ + +5 rows in set. Elapsed: 0.002 sec. + +:) SELECT s, arr, a, num, arrayEnumerate(arr) FROM arrays_test ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num + +SELECT s, arr, a, num, arrayEnumerate(arr) +FROM arrays_test +ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num + +┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ +│ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ +│ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ +│ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ +│ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ +│ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ +└───────┴─────────┴───┴─────┴─────────────────────┘ + +5 rows in set. Elapsed: 0.002 sec. +``` + +ARRAY JOIN also works with nested data structures. Example: + +```text +:) CREATE TABLE nested_test (s String, nest Nested(x UInt8, y UInt32)) ENGINE = Memory + +CREATE TABLE nested_test +( + s String, + nest Nested( + x UInt8, + y UInt32) +) ENGINE = Memory + +Ok. + +0 rows in set. Elapsed: 0.006 sec. + +:) INSERT INTO nested_test VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []) + +INSERT INTO nested_test VALUES + +Ok. + +3 rows in set. Elapsed: 0.001 sec. + +:) SELECT * FROM nested_test + +SELECT * +FROM nested_test + +┌─s───────┬─nest.x──┬─nest.y─────┐ +│ Hello │ [1,2] │ [10,20] │ +│ World │ [3,4,5] │ [30,40,50] │ +│ Goodbye │ [] │ [] │ +└─────────┴─────────┴────────────┘ + +3 rows in set. Elapsed: 0.001 sec. + +:) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest + +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest + +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +When specifying names of nested data structures in ARRAY JOIN, the meaning is the same as ARRAY JOIN with all the array elements that it consists of. Example: + +```text +:) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest.x, nest.y + +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x`, `nest.y` + +┌─s─────┬─nest.x─┬─nest.y─┐ +│ Hello │ 1 │ 10 │ +│ Hello │ 2 │ 20 │ +│ World │ 3 │ 30 │ +│ World │ 4 │ 40 │ +│ World │ 5 │ 50 │ +└───────┴────────┴────────┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +This variation also makes sense: + +```text +:) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest.x + +SELECT s, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN `nest.x` + +┌─s─────┬─nest.x─┬─nest.y─────┐ +│ Hello │ 1 │ [10,20] │ +│ Hello │ 2 │ [10,20] │ +│ World │ 3 │ [30,40,50] │ +│ World │ 4 │ [30,40,50] │ +│ World │ 5 │ [30,40,50] │ +└───────┴────────┴────────────┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +An alias may be used for a nested data structure, in order to select either the JOIN result or the source array. Example: + +```text +:) SELECT s, n.x, n.y, nest.x, nest.y FROM nested_test ARRAY JOIN nest AS n + +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` +FROM nested_test +ARRAY JOIN nest AS n + +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ +└───────┴─────┴─────┴─────────┴────────────┘ + +5 rows in set. Elapsed: 0.001 sec. +``` + +Example of using the arrayEnumerate function: + +```text +:) SELECT s, n.x, n.y, nest.x, nest.y, num FROM nested_test ARRAY JOIN nest AS n, arrayEnumerate(nest.x) AS num + +SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num +FROM nested_test +ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num + +┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ +│ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ +│ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ +│ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ +│ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ +│ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ +└───────┴─────┴─────┴─────────┴────────────┴─────┘ + +5 rows in set. Elapsed: 0.002 sec. +``` + +The query can only specify a single ARRAY JOIN clause. + +The corresponding conversion can be performed before the WHERE/PREWHERE clause (if its result is needed in this clause), or after completing WHERE/PREWHERE (to reduce the volume of calculations). + +### JOIN clause + +The normal JOIN, which is not related to ARRAY JOIN described above. + +```sql +[GLOBAL] ANY|ALL INNER|LEFT [OUTER] JOIN (subquery)|table USING columns_list +``` + +Performs joins with data from the subquery. At the beginning of query processing, the subquery specified after JOIN is run, and its result is saved in memory. Then it is read from the "left" table specified in the FROM clause, and while it is being read, for each of the read rows from the "left" table, rows are selected from the subquery results table (the "right" table) that meet the condition for matching the values of the columns specified in USING. + +The table name can be specified instead of a subquery. This is equivalent to the `SELECT * FROM table` subquery, except in a special case when the table has the Join engine – an array prepared for joining. + +All columns that are not needed for the JOIN are deleted from the subquery. + +There are several types of JOINs: + +`INNER` or `LEFT` type: +If INNER is specified, the result will contain only those rows that have a matching row in the right table. +If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything. + +`ANY` or `ALL` stringency:If `ANY` is specified and the right table has several matching rows, only the first one found is joined. +If `ALL` is specified and the right table has several matching rows, the data will be multiplied by the number of these rows. + +Using ALL corresponds to the normal JOIN semantic from standard SQL. +Using ANY is optimal. If the right table has only one matching row, the results of ANY and ALL are the same. You must specify either ANY or ALL (neither of them is selected by default). + +`GLOBAL` distribution: + +When using a normal JOIN, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. + +When using `GLOBAL ... JOIN`, first the requestor server runs a subquery to calculate the right table. This temporary table is passed to each remote server, and queries are run on them using the temporary data that was transmitted. + +Be careful when using GLOBAL JOINs. For more information, see the section "Distributed subqueries". + +Any combination of JOINs is possible. For example, `GLOBAL ANY LEFT OUTER JOIN`. + +When running a JOIN, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in WHERE and before aggregation. In order to explicitly set the processing order, we recommend running a JOIN subquery with a subquery. + +Example: + +```sql +SELECT + CounterID, + hits, + visits +FROM +( + SELECT + CounterID, + count() AS hits + FROM test.hits + GROUP BY CounterID +) ANY LEFT JOIN +( + SELECT + CounterID, + sum(Sign) AS visits + FROM test.visits + GROUP BY CounterID +) USING CounterID +ORDER BY hits DESC +LIMIT 10 +``` + +```text +┌─CounterID─┬───hits─┬─visits─┐ +│ 1143050 │ 523264 │ 13665 │ +│ 731962 │ 475698 │ 102716 │ +│ 722545 │ 337212 │ 108187 │ +│ 722889 │ 252197 │ 10547 │ +│ 2237260 │ 196036 │ 9522 │ +│ 23057320 │ 147211 │ 7689 │ +│ 722818 │ 90109 │ 17847 │ +│ 48221 │ 85379 │ 4652 │ +│ 19762435 │ 77807 │ 7026 │ +│ 722884 │ 77492 │ 11056 │ +└───────────┴────────┴────────┘ +``` + +Subqueries don't allow you to set names or use them for referencing a column from a specific subquery. +The columns specified in USING must have the same names in both subqueries, and the other columns must be named differently. You can use aliases to change the names of columns in subqueries (the example uses the aliases 'hits' and 'visits'). + +The USING clause specifies one or more columns to join, which establishes the equality of these columns. The list of columns is set without brackets. More complex join conditions are not supported. + +The right table (the subquery result) resides in RAM. If there isn't enough memory, you can't run a JOIN. + +Only one JOIN can be specified in a query (on a single level). To run multiple JOINs, you can put them in subqueries. + +Each time a query is run with the same JOIN, the subquery is run again – the result is not cached. To avoid this, use the special 'Join' table engine, which is a prepared array for joining that is always in RAM. For more information, see the section "Table engines, Join". + +In some cases, it is more efficient to use IN instead of JOIN. +Among the various types of JOINs, the most efficient is ANY LEFT JOIN, then ANY INNER JOIN. The least efficient are ALL LEFT JOIN and ALL INNER JOIN. + +If you need a JOIN for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a JOIN might not be very convenient due to the bulky syntax and the fact that the right table is re-accessed for every query. For such cases, there is an "external dictionaries" feature that you should use instead of JOIN. For more information, see the section "External dictionaries". + +### WHERE clause + +If there is a WHERE clause, it must contain an expression with the UInt8 type. This is usually an expression with comparison and logical operators. +This expression will be used for filtering data before all other transformations. + +If indexes are supported by the database table engine, the expression is evaluated on the ability to use indexes. + +### PREWHERE clause + +This clause has the same meaning as the WHERE clause. The difference is in which data is read from the table. +When using PREWHERE, first only the columns necessary for executing PREWHERE are read. Then the other columns are read that are needed for running the query, but only those blocks where the PREWHERE expression is true. + +It makes sense to use PREWHERE if there are filtration conditions that are not suitable for indexes that are used by a minority of the columns in the query, but that provide strong data filtration. This reduces the volume of data to read. + +For example, it is useful to write PREWHERE for queries that extract a large number of columns, but that only have filtration for a few columns. + +PREWHERE is only supported by tables from the `*MergeTree` family. + +A query may simultaneously specify PREWHERE and WHERE. In this case, PREWHERE precedes WHERE. + +Keep in mind that it does not make much sense for PREWHERE to only specify those columns that have an index, because when using an index, only the data blocks that match the index are read. + +If the 'optimize_move_to_prewhere' setting is set to 1 and PREWHERE is omitted, the system uses heuristics to automatically move parts of expressions from WHERE to PREWHERE. + +### GROUP BY clause + +This is one of the most important parts of a column-oriented DBMS. + +If there is a GROUP BY clause, it must contain a list of expressions. Each expression will be referred to here as a "key". +All the expressions in the SELECT, HAVING, and ORDER BY clauses must be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. + +If a query contains only table columns inside aggregate functions, the GROUP BY clause can be omitted, and aggregation by an empty set of keys is assumed. + +Example: + +```sql +SELECT + count(), + median(FetchTiming > 60 ? 60 : FetchTiming), + count() - sum(Refresh) +FROM hits +``` + +However, in contrast to standard SQL, if the table doesn't have any rows (either there aren't any at all, or there aren't any after using WHERE to filter), an empty result is returned, and not the result from one of the rows containing the initial values of aggregate functions. + +As opposed to MySQL (and conforming to standard SQL), you can't get some value of some column that is not in a key or aggregate function (except constant expressions). To work around this, you can use the 'any' aggregate function (get the first encountered value) or 'min/max'. + +Example: + +```sql +SELECT + domainWithoutWWW(URL) AS domain, + count(), + any(Title) AS title -- для каждого домена достаём первый попавшийся заголовок страницы +FROM hits +GROUP BY domain +``` + +For every different key value encountered, GROUP BY calculates a set of aggregate function values. + +GROUP BY is not supported for array columns. + +A constant can't be specified as arguments for aggregate functions. Example: sum(1). Instead of this, you can get rid of the constant. Example: `count()`. + +#### WITH TOTALS modifier + +If the WITH TOTALS modifier is specified, another row will be calculated. This row will have key columns containing default values (zeros or empty lines), and columns of aggregate functions with the values calculated across all the rows (the "total" values). + +This extra row is output in JSON\*, TabSeparated\*, and Pretty\* formats, separately from the other rows. In the other formats, this row is not output. + +In JSON\* formats, this row is output as a separate 'totals' field. In TabSeparated\* formats, the row comes after the main result, preceded by an empty row (after the other data). In Pretty\* formats, the row is output as a separate table after the main result. + +`WITH TOTALS` can be run in different ways when HAVING is present. The behavior depends on the 'totals_mode' setting. +By default, `totals_mode = 'before_having'`. In this case, 'totals' is calculated across all rows, including the ones that don't pass through HAVING and 'max_rows_to_group_by'. + +The other alternatives include only the rows that pass through HAVING in 'totals', and behave differently with the setting `max_rows_to_group_by` and `group_by_overflow_mode = 'any'`. + +`after_having_exclusive` – Don't include rows that didn't pass through `max_rows_to_group_by`. In other words, 'totals' will have less than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_inclusive` – Include all the rows that didn't pass through 'max_rows_to_group_by' in 'totals'. In other words, 'totals' will have more than or the same number of rows as it would if `max_rows_to_group_by` were omitted. + +`after_having_auto` – Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through 'max_rows_to_group_by' in 'totals'. Otherwise, do not include them. + +`totals_auto_threshold` – By default, 0.5. The coefficient for `after_having_auto`. + +If `max_rows_to_group_by` and `group_by_overflow_mode = 'any'` are not used, all variations of `after_having` are the same, and you can use any of them (for example, `after_having_auto`). + +You can use WITH TOTALS in subqueries, including subqueries in the JOIN clause (in this case, the respective total values are combined). + +#### GROUP BY in external memory + +You can enable dumping temporary data to the disk to restrict memory usage during GROUP BY. +The `max_bytes_before_external_group_by` setting determines the threshold RAM consumption for dumping GROUP BY temporary data to the file system. If set to 0 (the default), it is disabled. + +When using `max_bytes_before_external_group_by`, we recommend that you set max_memory_usage about twice as high. This is necessary because there are two stages to aggregation: reading the date and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn't dumped, then stage 2 might require up to the same amount of memory as in stage 1. + +For example, if `max_memory_usage` was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and max_memory_usage to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than ` max_bytes_before_external_group_by`. + +With distributed query processing, external aggregation is performed on remote servers. In order for the requestor server to use only a small amount of RAM, set ` distributed_aggregation_memory_efficient` to 1. + +When merging data flushed to the disk, as well as when merging results from remote servers when the ` distributed_aggregation_memory_efficient` setting is enabled, consumes up to 1/256 \* the number of threads from the total amount of RAM. + +When external aggregation is enabled, if there was less than ` max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). + +If you have an ORDER BY with a small LIMIT after GROUP BY, then the ORDER BY CLAUSE will not use significant amounts of RAM. +But if the ORDER BY doesn't have LIMIT, don't forget to enable external sorting (`max_bytes_before_external_sort`). + +### LIMIT N BY clause + +LIMIT N BY COLUMNS selects the top N rows for each group of COLUMNS. LIMIT N BY is not related to LIMIT; they can both be used in the same query. The key for LIMIT N BY can contain any number of columns or expressions. + +Example: + +```sql +SELECT + domainWithoutWWW(URL) AS domain, + domainWithoutWWW(REFERRER_URL) AS referrer, + device_type, + count() cnt +FROM hits +GROUP BY domain, referrer, device_type +ORDER BY cnt DESC +LIMIT 5 BY domain, device_type +LIMIT 100 +``` + +The query will select the top 5 referrers for each `domain, device_type` pair, but not more than 100 rows (`LIMIT n BY + LIMIT`). + +### HAVING clause + +Allows filtering the result received after GROUP BY, similar to the WHERE clause. +WHERE and HAVING differ in that WHERE is performed before aggregation (GROUP BY), while HAVING is performed after it. +If aggregation is not performed, HAVING can't be used. + + + +### ORDER BY clause + +The ORDER BY clause contains a list of expressions, which can each be assigned DESC or ASC (the sorting direction). If the direction is not specified, ASC is assumed. ASC is sorted in ascending order, and DESC in descending order. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` + +For sorting by String values, you can specify collation (comparison). Example: `ORDER BY SearchPhrase COLLATE 'tr'` - for sorting by keyword in ascending order, using the Turkish alphabet, case insensitive, assuming that strings are UTF-8 encoded. COLLATE can be specified or not for each expression in ORDER BY independently. If ASC or DESC is specified, COLLATE is specified after it. When using COLLATE, sorting is always case-insensitive. + +We only recommend using COLLATE for final sorting of a small number of rows, since sorting with COLLATE is less efficient than normal sorting by bytes. + +Rows that have identical values for the list of sorting expressions are output in an arbitrary order, which can also be nondeterministic (different each time). +If the ORDER BY clause is omitted, the order of the rows is also undefined, and may be nondeterministic as well. + +When floating point numbers are sorted, NaNs are separate from the other values. Regardless of the sorting order, NaNs come at the end. In other words, for ascending sorting they are placed as if they are larger than all the other numbers, while for descending sorting they are placed as if they are smaller than the rest. + +Less RAM is used if a small enough LIMIT is specified in addition to ORDER BY. Otherwise, the amount of memory spent is proportional to the volume of data for sorting. For distributed query processing, if GROUP BY is omitted, sorting is partially done on remote servers, and the results are merged on the requestor server. This means that for distributed sorting, the volume of data to sort can be greater than the amount of memory on a single server. + +If there is not enough RAM, it is possible to perform sorting in external memory (creating temporary files on a disk). Use the setting `max_bytes_before_external_sort` for this purpose. If it is set to 0 (the default), external sorting is disabled. If it is enabled, when the volume of data to sort reaches the specified number of bytes, the collected data is sorted and dumped into a temporary file. After all data is read, all the sorted files are merged and the results are output. Files are written to the /var/lib/clickhouse/tmp/ directory in the config (by default, but you can use the 'tmp_path' parameter to change this setting). + +Running a query may use more memory than 'max_bytes_before_external_sort'. For this reason, this setting must have a value significantly smaller than 'max_memory_usage'. As an example, if your server has 128 GB of RAM and you need to run a single query, set 'max_memory_usage' to 100 GB, and 'max_bytes_before_external_sort' to 80 GB. + +External sorting works much less effectively than sorting in RAM. + +### SELECT clause + +The expressions specified in the SELECT clause are analyzed after the calculations for all the clauses listed above are completed. +More specifically, expressions are analyzed that are above the aggregate functions, if there are any aggregate functions. +The aggregate functions and everything below them are calculated during aggregation (GROUP BY). +These expressions work as if they are applied to separate rows in the result. + +### DISTINCT clause + +If DISTINCT is specified, only a single row will remain out of all the sets of fully matching rows in the result. +The result will be the same as if GROUP BY were specified across all the fields specified in SELECT without aggregate functions. But there are several differences from GROUP BY: + +- DISTINCT can be applied together with GROUP BY. +- When ORDER BY is omitted and LIMIT is defined, the query stops running immediately after the required number of different rows has been read. +- Data blocks are output as they are processed, without waiting for the entire query to finish running. + +DISTINCT is not supported if SELECT has at least one array column. + +### LIMIT clause + +LIMIT m allows you to select the first 'm' rows from the result. +LIMIT n, m allows you to select the first 'm' rows from the result after skipping the first 'n' rows. + +'n' and 'm' must be non-negative integers. + +If there isn't an ORDER BY clause that explicitly sorts results, the result may be arbitrary and nondeterministic. + +### UNION ALL clause + +You can use UNION ALL to combine any number of queries. Example: + +```sql +SELECT CounterID, 1 AS table, toInt64(count()) AS c + FROM test.hits + GROUP BY CounterID + +UNION ALL + +SELECT CounterID, 2 AS table, sum(Sign) AS c + FROM test.visits + GROUP BY CounterID + HAVING c > 0 +``` + +Only UNION ALL is supported. The regular UNION (UNION DISTINCT) is not supported. If you need UNION DISTINCT, you can write SELECT DISTINCT from a subquery containing UNION ALL. + +Queries that are parts of UNION ALL can be run simultaneously, and their results can be mixed together. + +The structure of results (the number and type of columns) must match for the queries. But the column names can differ. In this case, the column names for the final result will be taken from the first query. + +Queries that are parts of UNION ALL can't be enclosed in brackets. ORDER BY and LIMIT are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with UNION ALL in a subquery in the FROM clause. + +### INTO OUTFILE clause + +Add the `INTO OUTFILE filename` clause (where filename is a string literal) to redirect query output to the specified file. +In contrast to MySQL, the file is created on the client side. The query will fail if a file with the same filename already exists. +This functionality is available in the command-line client and clickhouse-local (a query sent via HTTP interface will fail). + +The default output format is TabSeparated (the same as in the command-line client batch mode). + +### FORMAT clause + +Specify 'FORMAT format' to get data in any specified format. +You can use this for convenience, or for creating dumps. +For more information, see the section "Formats". +If the FORMAT clause is omitted, the default format is used, which depends on both the settings and the interface used for accessing the DB. For the HTTP interface and the command-line client in batch mode, the default format is TabSeparated. For the command-line client in interactive mode, the default format is PrettyCompact (it has attractive and compact tables). + +When using the command-line client, data is passed to the client in an internal efficient format. The client independently interprets the FORMAT clause of the query and formats the data itself (thus relieving the network and the server from the load). + +### IN operators + +The `IN`, `NOT IN`, `GLOBAL IN`, and `GLOBAL NOT IN` operators are covered separately, since their functionality is quite rich. + +The left side of the operator is either a single column or a tuple. + +Examples: + +```sql +SELECT UserID IN (123, 456) FROM ... +SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... +``` + +If the left side is a single column that is in the index, and the right side is a set of constants, the system uses the index for processing the query. + +Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section "External data for query processing"), then use a subquery. + +The right side of the operator can be a set of constant expressions, a set of tuples with constant expressions (shown in the examples above), or the name of a database table or SELECT subquery in brackets. + +If the right side of the operator is the name of a table (for example, `UserID IN users`), this is equivalent to the subquery `UserID IN (SELECT * FROM users)`. Use this when working with external data that is sent along with the query. For example, the query can be sent together with a set of user IDs loaded to the 'users' temporary table, which should be filtered. + +If the right side of the operator is a table name that has the Set engine (a prepared data set that is always in RAM), the data set will not be created over again for each query. + +The subquery may specify more than one column for filtering tuples. +Example: + +```sql +SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... +``` + +The columns to the left and right of the IN operator should have the same type. + +The IN operator and subquery may occur in any part of the query, including in aggregate functions and lambda functions. +Example: + +```sql +SELECT + EventDate, + avg(UserID IN + ( + SELECT UserID + FROM test.hits + WHERE EventDate = toDate('2014-03-17') + )) AS ratio +FROM test.hits +GROUP BY EventDate +ORDER BY EventDate ASC +``` + +```text +┌──EventDate─┬────ratio─┐ +│ 2014-03-17 │ 1 │ +│ 2014-03-18 │ 0.807696 │ +│ 2014-03-19 │ 0.755406 │ +│ 2014-03-20 │ 0.723218 │ +│ 2014-03-21 │ 0.697021 │ +│ 2014-03-22 │ 0.647851 │ +│ 2014-03-23 │ 0.648416 │ +└────────────┴──────────┘ +``` + +For each day after March 17th, count the percentage of pageviews made by users who visited the site on March 17th. +A subquery in the IN clause is always run just one time on a single server. There are no dependent subqueries. + + + +#### Distributed subqueries + +There are two options for IN-s with subqueries (similar to JOINs): normal `IN` / ` OIN` and `IN GLOBAL` / `GLOBAL JOIN`. They differ in how they are run for distributed query processing. + +
+ +Remember that the algorithms described below may work differently depending on the [](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting. + +
+ +When using the regular IN, the query is sent to remote servers, and each of them runs the subqueries in the `IN` or `JOIN` clause. + +When using `GLOBAL IN` / `GLOBAL JOINs`, first all the subqueries are run for `GLOBAL IN` / `GLOBAL JOINs`, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. + +For a non-distributed query, use the regular `IN` / `JOIN`. + +Be careful when using subqueries in the `IN` / `JOIN` clauses for distributed query processing. + +Let's look at some examples. Assume that each server in the cluster has a normal **local_table**. Each server also has a **distributed_table** table with the **Distributed** type, which looks at all the servers in the cluster. + +For a query to the **distributed_table**, the query will be sent to all the remote servers and run on them using the **local_table**. + +For example, the query + +```sql +SELECT uniq(UserID) FROM distributed_table +``` + +will be sent to all remote servers as + +```sql +SELECT uniq(UserID) FROM local_table +``` + +and run on each of them in parallel, until it reaches the stage where intermediate results can be combined. Then the intermediate results will be returned to the requestor server and merged on it, and the final result will be sent to the client. + +Now let's examine a query with IN: + +```sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +- Calculation of the intersection of audiences of two sites. + +This query will be sent to all remote servers as + +```sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) +``` + +In other words, the data set in the IN clause will be collected on each server independently, only across the data that is stored locally on each of the servers. + +This will work correctly and optimally if you are prepared for this case and have spread data across the cluster servers such that the data for a single UserID resides entirely on a single server. In this case, all the necessary data will be available locally on each server. Otherwise, the result will be inaccurate. We refer to this variation of the query as "local IN". + +To correct how the query works when data is spread randomly across the cluster servers, you could specify **distributed_table** inside a subquery. The query would look like this: + +```sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +This query will be sent to all remote servers as + +```sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The subquery will begin running on each remote server. Since the subquery uses a distributed table, the subquery that is on each remote server will be resent to every remote server as + +```sql +SELECT UserID FROM local_table WHERE CounterID = 34 +``` + +For example, if you have a cluster of 100 servers, executing the entire query will require 10,000 elementary requests, which is generally considered unacceptable. + +In such cases, you should always use GLOBAL IN instead of IN. Let's look at how it works for the query + +```sql +SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) +``` + +The requestor server will run the subquery + +```sql +SELECT UserID FROM distributed_table WHERE CounterID = 34 +``` + +and the result will be put in a temporary table in RAM. Then the request will be sent to each remote server as + +```sql +SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 +``` + +and the temporary table '_data1' will be sent to every remote server together with the query (the name of the temporary table is implementation-defined). + +This is more optimal than using the normal IN. However, keep the following points in mind: + +1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don't need to do this for a normal IN.) +2. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. +3. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. +4. Try to distribute data across servers so that you don't need to use GLOBAL IN on a regular basis. +5. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that a single group of replicas resides in no more than one data center with a fast network between them, so that a query can be processed entirely within a single data center. + +It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. + +### Extreme values + +In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. + +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in JSON\*, TabSeparated\*, and Pretty\* formats, separate from the other rows. They are not output for other formats. + +In JSON\* formats, the extreme values are output in a separate 'extremes' field. In TabSeparated\* formats, the row comes after the main result, and after 'totals' if present. It is preceded by an empty row (after the other data). In Pretty\* formats, the row is output as a separate table after the main result, and after 'totals' if present. + +Extreme values are calculated for rows that have passed through LIMIT. However, when using 'LIMIT offset, size', the rows before 'offset' are included in 'extremes'. In stream requests, the result may also include a small number of rows that passed through LIMIT. + +### Notes + +The `GROUP BY` and `ORDER BY` clauses do not support positional arguments. This contradicts MySQL, but conforms to standard SQL. +For example, `GROUP BY 1, 2` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). + +You can use synonyms (`AS` aliases) in any part of a query. + +You can put an asterisk in any part of a query instead of an expression. When the query is analyzed, the asterisk is expanded to a list of all table columns (excluding the `MATERIALIZED` and `ALIAS` columns). There are only a few cases when using an asterisk is justified: + +- When creating a table dump. +- For tables containing just a few columns, such as system tables. +- For getting information about what columns are in a table. In this case, set `LIMIT 1`. But it is better to use the `DESC TABLE` query. +- When there is strong filtration on a small number of columns using `PREWHERE`. +- In subqueries (since columns that aren't needed for the external query are excluded from subqueries). + +In all other cases, we don't recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. + +## KILL QUERY + +```sql +KILL QUERY WHERE [SYNC|ASYNC|TEST] [FORMAT format] +``` + +Attempts to terminate queries currently running. +The queries to terminate are selected from the system.processes table for which expression_for_system.processes is true. + +Examples: + +```sql +KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' +``` + +Terminates all queries with the specified query_id. + +```sql +KILL QUERY WHERE user='username' SYNC +``` + +Synchronously terminates all queries run by `username`. + +Readonly-users can only terminate their own requests. +By default, the asynchronous version of queries is used (`ASYNC`), which terminates without waiting for queries to complete. +The synchronous version (`SYNC`) waits for all queries to be completed and displays information about each process as it terminates. +The response contains the `kill_status` column, which can take the following values: + +1. 'finished' – The query completed successfully. +2. 'waiting' – Waiting for the query to finish after sending it a signal to terminate. +3. The other values ​​explain why the query can't be terminated. + +A test query (`TEST`) only checks the user's rights and displays a list of queries to terminate. + diff --git a/docs/en/query_language/queries.rst b/docs/en/query_language/queries.rst deleted file mode 100644 index b22eefa0702..00000000000 --- a/docs/en/query_language/queries.rst +++ /dev/null @@ -1,1476 +0,0 @@ -Queries -------- - -CREATE DATABASE -~~~~~~~~~~~~~~~ -Creates the 'db_name' database. - -.. code-block:: sql - - CREATE DATABASE [IF NOT EXISTS] db_name - -A database is just a directory for tables. -If "IF NOT EXISTS" is included, the query won't return an error if the database already exists. - -CREATE TABLE -~~~~~~~~~~~~ -The ``CREATE TABLE`` query can have several forms. - -.. code-block:: sql - - CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] - ( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... - ) ENGINE = engine - -Creates a table named 'name' in the 'db' database or the current database if 'db' is not set, with the structure specified in brackets and the 'engine' engine. The structure of the table is a list of column descriptions. If indexes are supported by the engine, they are indicated as parameters for the table engine. - -A column description is ``name type`` in the simplest case. For example: ``RegionID UInt32``. -Expressions can also be defined for default values (see below). - -.. code-block:: sql - - CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name AS [db2.]name2 [ENGINE = engine] - -Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the 'db2.name2' table. - -.. code-block:: sql - - CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name ENGINE = engine AS SELECT ... - -Creates a table with a structure like the result of the ``SELECT`` query, with the 'engine' engine, and fills it with data from SELECT. - -In all cases, if IF NOT EXISTS is specified, the query won't return an error if the table already exists. In this case, the query won't do anything. - -Default values -"""""""""""""" -The column description can specify an expression for a default value, in one of the following ways: -``DEFAULT expr``, ``MATERIALIZED expr``, ``ALIAS expr``. -Example: ``URLDomain String DEFAULT domain(URL)``. - -If an expression for the default value is not defined, the default values will be set to zeros for numbers, empty strings for strings, empty arrays for arrays, and 0000-00-00 for dates or 0000-00-00 00:00:00 for dates with time. NULLs are not supported. - -If the default expression is defined, the column type is optional. If there isn't an explicitly defined type, the default expression type is used. Example: ``EventDate DEFAULT toDate(EventTime)`` - the 'Date' type will be used for the 'EventDate' column. - -If the data type and default expression are defined explicitly, this expression will be cast to the specified type using type casting functions. Example: ``Hits UInt32 DEFAULT 0`` means the same thing as ``Hits UInt32 DEFAULT toUInt32(0)``. - -Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don't contain loops. For INSERT, it checks that expressions are resolvable - that all columns they can be calculated from have been passed. - -``DEFAULT expr`` - -Normal default value. If the INSERT query doesn't specify the corresponding column, it will be filled in by computing the corresponding expression. - -``MATERIALIZED expr`` - -Materialized expression. Such a column can't be specified for INSERT, because it is always calculated. -For an INSERT without a list of columns, these columns are not considered. -In addition, this column is not substituted when using an asterisk in a SELECT query. This is to preserve the invariant that the dump obtained using SELECT * can be inserted back into the table using INSERT without specifying the list of columns. - -``ALIAS expr`` - -Synonym. Such a column isn't stored in the table at all. -Its values can't be inserted in a table, and it is not substituted when using an asterisk in a SELECT query. -It can be used in SELECTs if the alias is expanded during query parsing. - -When using the ALTER query to add new columns, old data for these columns is not written. Instead, when reading old data that does not have values for the new columns, expressions are computed on the fly by default. However, if running the expressions requires different columns that are not indicated in the query, these columns will additionally be read, but only for the blocks of data that need it. - -If you add a new column to a table but later change its default expression, the values used for old data will change (for data where values were not stored on the disk). Note that when running background merges, data for columns that are missing in one of the merging parts is written to the merged part. - -It is not possible to set default values for elements in nested data structures. - -Temporary tables -"""""""""""""""" -In all cases, if TEMPORARY is specified, a temporary table will be created. Temporary tables have the following characteristics: -- Temporary tables disappear when the session ends, including if the connection is lost. -- A temporary table is created with the Memory engine. The other table engines are not supported. -- The DB can't be specified for a temporary table. It is created outside of databases. -- If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used. -- For distributed query processing, temporary tables used in a query are passed to remote servers. - -In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed (GLOBAL) IN. For more information, see the appropriate sections. - -Distributed DDL queries (ON CLUSTER section) -"""""""""""""""""""""""""""""""""""""""""""" - -Queries ``CREATE``, ``DROP``, ``ALTER``, ``RENAME`` support distributed execution on cluster. -For example, the following query creates ``Distributed`` table ``all_hits`` for each host of the cluster ``cluster``: - -.. code-block:: sql - - CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, hits) - -To correctly execute such queries you need to have equal definitions of the cluster on each host (you can use :ref:`ZooKeeper substitutions ` to syncronize configs on hosts) and connection to ZooKeeper quorum. -Local version of the query will be eventually executed on each host of the cluster, even if some hosts are temporary unavaiable; on each host queries are executed stictly sequentually. -At the moment, ``ALTER`` queries for replicated tables are not supported yet. - -CREATE VIEW -~~~~~~~~~~~ -``CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ...`` - -Creates a view. There are two types of views: normal and MATERIALIZED. - -Normal views don't store any data, but just perform a read from another table. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the FROM clause. - -As an example, assume you've created a view: - -.. code-block:: sql - - CREATE VIEW view AS SELECT ... - -and written a query: - -.. code-block:: sql - - SELECT a, b, c FROM view - -This query is fully equivalent to using the subquery: - -.. code-block:: sql - - SELECT a, b, c FROM (SELECT ...) - -Materialized views store data transformed by the corresponding SELECT query. - -When creating a materialized view, you have to either specify ENGINE - the table engine for storing data, or target table for materialized results. By default, it uses the same engine as for the table that the SELECT query is made from. - -A materialized view is arranged as follows: when inserting data to the table specified in SELECT, part of the inserted data is converted by this SELECT query, and the result is inserted in the view. - -If you specify POPULATE, the existing table data is inserted in the view when creating it, as if making a CREATE TABLE ... AS SELECT ... query. Otherwise, the query contains only the data inserted in the table after creating the view. We don't recommend using POPULATE, since data inserted in the table during the view creation will not be inserted in it. - -The SELECT query can contain DISTINCT, GROUP BY, ORDER BY, LIMIT ... Note that the corresponding conversions are performed independently on each block of inserted data. For example, if GROUP BY is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won't be further aggregated. The exception is when using an ENGINE that independently performs data aggregation, such as SummingMergeTree. - -The execution of ALTER queries on materialized views has not been fully developed, so they might be inconvenient. -If the materialized view uses a ``TO [db.]name`` to specify a target table, it is possible to DETACH the view, ALTER the target table, and ATTACH the view again. - -Views look the same as normal tables. For example, they are listed in the result of the SHOW TABLES query. - -There isn't a separate query for deleting views. To delete a view, use DROP TABLE. - -ATTACH -~~~~~~ -The query is exactly the same as CREATE, except -- The word ATTACH is used instead of CREATE. -- The query doesn't create data on the disk, but assumes that data is already in the appropriate places, and just adds information about the table to the server. -After executing an ATTACH query, the server will know about the existence of the table. - -If the table has been previously detached and it's structure is known, it's possible to use shorthand form and omit structure definition: - -.. code-block:: sql - - ATTACH TABLE [IF NOT EXISTS] [db.]name - -This query is used when starting the server. The server stores table metadata as files with ATTACH queries, which it simply runs at launch (with the exception of system tables, which are explicitly created on the server). - -DROP -~~~~ -This query has two types: ``DROP DATABASE`` and ``DROP TABLE``. - -.. code-block:: sql - - DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] - -Deletes all tables inside the 'db' database, then deletes the 'db' database itself. -If IF EXISTS is specified, it doesn't return an error if the database doesn't exist. - -.. code-block:: sql - - DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] - -Deletes the table. -If ``IF EXISTS`` is specified, it doesn't return an error if the table doesn't exist or the database doesn't exist. - -DETACH -~~~~~~ -Deletes information about the table from the server. The server stops knowing about the table's existence. - -.. code-block:: sql - - DETACH TABLE [IF EXISTS] [db.]name - -This does not delete the table's data or metadata. On the next server launch, the server will read the metadata and find out about the table again. Similarly, a "detached" table can be re-attached using the ATTACH query (with the exception of system tables, which do not have metadata stored for them). - -There is no DETACH DATABASE query. - -RENAME -~~~~~~ -Renames one or more tables. - -.. code-block:: sql - - RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... [ON CLUSTER cluster] - -All tables are renamed under global locking. Renaming tables is a light operation. If you indicated another database after TO, the table will be moved to this database. However, the directories with databases must reside in the same file system (otherwise, an error is returned). - -ALTER -~~~~~ -The ALTER query is only supported for \*MergeTree type tables, as well as for Merge and Distributed types. The query has several variations. - -Column manipulations -"""""""""""""""""""""""" -Lets you change the table structure. - -.. code-block:: sql - - ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|MODIFY COLUMN ... - -In the query, specify a list of one or more comma-separated actions. Each action is an operation on a column. - -The following actions are supported: - -.. code-block:: sql - - ADD COLUMN name [type] [default_expr] [AFTER name_after] - -Adds a new column to the table with the specified name, type, and default expression (see the section "Default expressions"). If you specify 'AFTER name_after' (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, 'name_after' can be the name of a column that is added in one of the previous actions. - -Adding a column just changes the table structure, without performing any actions with data. The data doesn't appear on the disk after ALTER. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see MergeTree). - -This approach allows us to complete the ALTER query instantly, without increasing the volume of old data. - -.. code-block:: sql - - DROP COLUMN name - -Deletes the column with the name 'name'. - -Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly. - -.. code-block:: sql - - MODIFY COLUMN name [type] [default_expr] - -Changes the 'name' column's type to 'type' and/or the default expression to 'default_expr'. When changing the type, values are converted as if the 'toType' function were applied to them. - -If only the default expression is changed, the query doesn't do anything complex, and is completed almost instantly. - -Changing the column type is the only complex action - it changes the contents of files with data. For large tables, this may take a long time. - -There are several stages of execution: -- Preparing temporary (new) files with modified data. -- Renaming old files. -- Renaming the temporary (new) files to the old names. -- Deleting the old files. - -Only the first stage takes time. If there is a failure at this stage, the data is not changed. -If there is a failure during one of the successive stages, data can be restored manually. The exception is if the old files were deleted from the file system but the data for the new files did not get written to the disk and was lost. - -There is no support for changing the column type in arrays and nested data structures. - -The ALTER query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like 'name.nested_name' and the type 'Array(T)'. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot. - -There is no support for deleting of columns in the primary key or the sampling key (columns that are in the ENGINE expression). Changing the type of columns in the primary key is allowed only if such change doesn't entail changing the actual data (e.g. adding the value to an Enum or changing the type from DateTime to UInt32 is allowed). - -If the ALTER query is not sufficient for making the table changes you need, you can create a new table, copy the data to it using the INSERT SELECT query, then switch the tables using the RENAME query and delete the old table. - -The ALTER query blocks all reads and writes for the table. In other words, if a long SELECT is running at the time of the ALTER query, the ALTER query will wait for the SELECT to complete. At the same time, all new queries to the same table will wait while this ALTER is running. - -For tables that don't store data themselves (Merge and Distributed), ALTER just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a Distributed table, you will also need to run ALTER for the tables on all remote servers. - -The ALTER query for changing columns is replicated. The instructions are saved in ZooKeeper, then each replica applies them. All ALTER queries are run in the same order. The query waits for the appropriate actions to be completed on the other replicas. However, a query to change columns in a replicated table can be interrupted, and all actions will be performed asynchronously. - -Manipulations with partitions and parts -""""""""""""""""""""""""""""""""""""""" -Only works for tables in the MergeTree family. The following operations are available: - -* ``DETACH PARTITION`` - Move a partition to the 'detached' directory and forget it. -* ``DROP PARTITION`` - Delete a partition. -* ``ATTACH PART|PARTITION`` - Add a new part or partition from the 'detached' directory to the table. -* ``FREEZE PARTITION`` - Create a backup of a partition. -* ``FETCH PARTITION`` - Download a partition from another server. - -Each type of query is covered separately below. - -A partition in a table is data for a single calendar month. This is determined by the values of the date key specified in the table engine parameters. Each month's data is stored separately in order to simplify manipulations with this data. - -A "part" in the table is part of the data from a single partition, sorted by the primary key. - -You can use the ``system.parts`` table to view the set of table parts and partitions: - -.. code-block:: text - - SELECT * FROM system.parts WHERE active - -``active`` - Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part - these parts are deleted approximately 10 minutes after merging. - -Another way to view a set of parts and partitions is to go into the directory with table data. -The directory with data is -/var/lib/clickhouse/data/database/table/, -where /var/lib/clickhouse/ is the path to ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: - -.. code-block:: bash - - $ ls -l /var/lib/clickhouse/data/test/visits/ - total 48 - drwxrwxrwx 2 clickhouse clickhouse 20480 мая 13 02:58 20140317_20140323_2_2_0 - drwxrwxrwx 2 clickhouse clickhouse 20480 мая 13 02:58 20140317_20140323_4_4_0 - drwxrwxrwx 2 clickhouse clickhouse 4096 мая 13 02:55 detached - -rw-rw-rw- 1 clickhouse clickhouse 2 мая 13 02:58 increment.txt - -Here ``20140317_20140323_2_2_0``, ``20140317_20140323_4_4_0`` - are directories of parts. - -Let's look at the name of the first part: ``20140317_20140323_2_2_0``. -* ``20140317`` - minimum date of part data -* ``20140323`` - maximum date of part data -* ``2`` - minimum number of the data block -* ``2`` - maximum number of the data block -* ``0`` - part level, depth of the merge tree that formed it - -Each part corresponds to a single partition and contains data for a single month. -201403 - The partition name. A partition is a set of parts for a single month. - -On an operating server, you can't manually change the set of parts or their data on the file system, since the server won't know about it. For non-replicated tables, you can do this when the server is stopped, but we don't recommended it. For replicated tables, the set of parts can't be changed in any case. - -The 'detached' directory contains parts that are not used by the server - detached from the table using the ALTER ... DETACH query. Parts that are damaged are also moved to this directory, instead of deleting them. You can add, delete, or modify the data in the 'detached' directory at any time - the server won't know about this until you make the ALTER TABLE ... ATTACH query. - -.. code-block:: sql - - ALTER TABLE [db.]table DETACH PARTITION 'name' - -Move all data for partitions named 'name' to the 'detached' directory and forget about them. -The partition name is specified in YYYYMM format. It can be indicated in single quotes or without them. - -After the query is executed, you can do whatever you want with the data in the 'detached' directory — delete it from the file system, or just leave it. - -The query is replicated - data will be moved to the 'detached' directory and forgotten on all replicas. The query can only be sent to a leader replica. To find out if a replica is a leader, perform SELECT to the 'system.replicas' system table. Alternatively, it is easier to make a query on all replicas, and all except one will throw an exception. - -.. code-block:: sql - - ALTER TABLE [db.]table DROP PARTITION 'name' - -Similar to the DETACH operation. Deletes data from the table. Data parts will be tagged as inactive and will be completely deleted in approximately 10 minutes. The query is replicated - data will be deleted on all replicas. - -.. code-block:: sql - - ALTER TABLE [db.]table ATTACH PARTITION|PART 'name' - -Adds data to the table from the 'detached' directory. - -It is possible to add data for an entire partition or a separate part. For a part, specify the full name of the part in single quotes. - -The query is replicated. Each replica checks whether there is data in the 'detached' directory. If there is data, it checks the integrity, verifies that it matches the data on the server that initiated the query, and then adds it if everything is correct. If not, it downloads data from the query requestor replica, or from another replica where the data has already been added. - -So you can put data in the 'detached' directory on one replica, and use the ALTER ... ATTACH query to add it to the table on all replicas. - -.. code-block:: sql - - ALTER TABLE [db.]table FREEZE PARTITION 'name' - -Creates a local backup of one or multiple partitions. The name can be the full name of the partition (for example, 201403), or its prefix (for example, 2014) - then the backup will be created for all the corresponding partitions. - -The query does the following: for a data snapshot at the time of execution, it creates hardlinks to table data in the directory /var/lib/clickhouse/shadow/N/... -/var/lib/clickhouse/ is the working ClickHouse directory from the config. -N is the incremental number of the backup. - -``/var/lib/clickhouse/`` - working directory of ClickHouse from config file. -``N`` - incremental number of backup. - -The same structure of directories is created inside the backup as inside ``/var/lib/clickhouse/``. -It also performs 'chmod' for all files, forbidding writes to them. - -The backup is created almost instantly (but first it waits for current queries to the corresponding table to finish running). At first, the backup doesn't take any space on the disk. As the system works, the backup can take disk space, as data is modified. If the backup is made for old enough data, it won't take space on the disk. - -After creating the backup, data from ``/var/lib/clickhouse/shadow/`` can be copied to the remote server and then deleted on the local server. The entire backup process is performed without stopping the server. - -The ``ALTER ... FREEZE PARTITION`` query is not replicated. A local backup is only created on the local server. - -As an alternative, you can manually copy data from the ``/var/lib/clickhouse/data/database/table directory``. But if you do this while the server is running, race conditions are possible when copying directories with files being added or changed, and the backup may be inconsistent. You can do this if the server isn't running - then the resulting data will be the same as after the ALTER TABLE t FREEZE PARTITION query. - -``ALTER TABLE ... FREEZE PARTITION`` only copies data, not table metadata. To make a backup of table metadata, copy the file ``/var/lib/clickhouse/metadata/database/table.sql`` - -To restore from a backup: - -* Use the CREATE query to create the table if it doesn't exist. The query can be taken from an .sql file (replace ATTACH in it with CREATE). -* Copy data from the ``data/database/table/`` directory inside the backup to the ``/var/lib/clickhouse/data/database/table/detached/`` directory. -* Run ``ALTER TABLE ... ATTACH PARTITION YYYYMM``queries where ``YYYYMM`` is the month, for every month. - -In this way, data from the backup will be added to the table. -Restoring from a backup doesn't require stopping the server. - -Backups and replication -""""""""""""""""""""""" -Replication provides protection from device failures. If all data disappeared on one of your replicas, follow the instructions in the "Restoration after failure" section to restore it. - -For protection from device failures, you must use replication. For more information about replication, see the section "Data replication". - -Backups protect against human error (accidentally deleting data, deleting the wrong data or in the wrong cluster, or corrupting data). For high-volume databases, it can be difficult to copy backups to remote servers. In such cases, to protect from human error, you can keep a backup on the same server (it will reside in /var/lib/clickhouse/shadow/). - -.. code-block:: sql - - ALTER TABLE [db.]table FETCH PARTITION 'name' FROM 'path-in-zookeeper' - -This query only works for replicatable tables. - -It downloads the specified partition from the shard that has its ZooKeeper path specified in the FROM clause, then puts it in the 'detached' directory for the specified table. - -Although the query is called ALTER TABLE, it does not change the table structure, and does not immediately change the data available in the table. - -Data is placed in the 'detached' directory. You can use the ALTER TABLE ... ATTACH query to attach the data. - -The path to ZooKeeper is specified in the FROM clause. For example, ``/clickhouse/tables/01-01/visits``. -Before downloading, the system checks that the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. - -The ALTER ... FETCH PARTITION query is not replicated. The partition will be downloaded to the 'detached' directory only on the local server. Note that if after this you use the ALTER TABLE ... ATTACH query to add data to the table, the data will be added on all replicas (on one of the replicas it will be added from the 'detached' directory, and on the rest it will be loaded from neighboring replicas). - -Synchronicity of ALTER queries -"""""""""""""""""""""""""""""" -For non-replicatable tables, all ALTER queries are performed synchronously. For replicatable tables, the query just adds instructions for the appropriate actions to ZooKeeper, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. - -For ``ALTER ... ATTACH|DETACH|DROP`` queries, you can use the ``'replication_alter_partitions_sync'`` setting to set up waiting. -Possible values: 0 - do not wait, 1 - wait for own completion (default), 2 - wait for all. - -SHOW DATABASES -~~~~~~~~~~~~~~ - -.. code-block:: sql - - SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] - -Prints a list of all databases. -This query is identical to the query ``SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`` -See the section "Formats". - -SHOW TABLES -~~~~~~~~~~~ - -.. code-block:: sql - - SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] - -Outputs a list of -* tables from the current database, or from the 'db' database if "FROM db" is specified. -* all tables, or tables whose name matches the pattern, if "LIKE 'pattern'" is specified. - -The query is identical to the query SELECT name FROM system.tables -WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] -See the section "LIKE operator". - -SHOW PROCESSLIST -~~~~~~~~~~~~~~~~ - -.. code-block:: sql - - SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] - -Outputs a list of queries currently being processed, other than SHOW PROCESSLIST queries. - -Prints a table containing the columns: - -**user** is the user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the 'default' user. SHOW PROCESSLIST shows the username for a specific query, not for a query that this query initiated. - -**address** is the name of the host that the query was sent from. For distributed processing, on remote servers, this is the name of the query requestor host. To track where a distributed query was originally made from, look at SHOW PROCESSLIST on the query requestor server. - -**elapsed** - The execution time, in seconds. Queries are output in order of decreasing execution time. - -**rows_read**, **bytes_read** - How many rows and bytes of uncompressed data were read when processing the query. For distributed processing, data is totaled from all the remote servers. This is the data used for restrictions and quotas. - -**memory_usage** - Current RAM usage in bytes. See the setting 'max_memory_usage'. - -**query** - The query itself. In INSERT queries, the data for insertion is not output. - -**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. - -This query is exactly the same as: SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]. - -Tip (execute in the console): -``watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"`` - -SHOW CREATE TABLE -~~~~~~~~~~~~~~~~~ - -.. code-block:: sql - - SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] - -Returns a single String-type 'statement' column, which contains a single value - the CREATE query used for creating the specified table. - -DESCRIBE TABLE -~~~~~~~~~~~~~~ - -.. code-block:: sql - - DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] - -Returns two String-type columns: 'name' and 'type', which indicate the names and types of columns in the specified table. - -Nested data structures are output in "expanded" format. Each column is shown separately, with the name after a dot. - -EXISTS -~~~~~~ - -.. code-block:: sql - - EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] - -Returns a single UInt8-type column, which contains the single value 0 if the table or database doesn't exist, or 1 if the table exists in the specified database. - -USE -~~~ - -.. code-block:: sql - - USE db - -Lets you set the current database for the session. -The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name. -This query can't be made when using the HTTP protocol, since there is no concept of a session. - -SET -~~~ - -.. code-block:: sql - - SET [GLOBAL] param = value - -Lets you set the 'param' setting to 'value'. You can also make all the settings from the specified settings profile in a single query. To do this, specify 'profile' as the setting name. For more information, see the section "Settings". The setting is made for the session, or for the server (globally) if GLOBAL is specified. -When making a global setting, the setting is not applied to sessions already running, including the current session. It will only be used for new sessions. - -When the server is restarted, settings made using SET are lost. -To make settings that persist after a server restart, you can only use the server's config file. - -OPTIMIZE -~~~~~~~~ - -.. code-block:: sql - - OPTIMIZE TABLE [db.]name [PARTITION partition] [FINAL] - -Asks the table engine to do something for optimization. -Supported only by \*MergeTree engines, in which this query initializes a non-scheduled merge of data parts. -If ``PARTITION`` is specified, then only specified partition will be optimized. -If ``FINAL`` is specified, then optimization will be performed even if data inside the partition already optimized (i. e. all data is in single part). - -INSERT -~~~~~~ -This query has several variations. - -.. code-block:: sql - - INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... - -Inserts rows with the listed values in the 'table' table. -This query is exactly the same as: - -.. code-block:: sql - - INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... - -.. code-block:: sql - - INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format ... - -Inserts data in any specified format. -The data itself comes after 'format', after all space symbols up to the first line break if there is one and including it, or after all space symbols if there isn't a line break. We recommend writing data starting from the next line (this is important if the data starts with space characters). - -Example: - -.. code-block:: sql - - INSERT INTO t FORMAT TabSeparated - 11 Hello, world! - 22 Qwerty - -For more information about data formats, see the section "Formats". The "Interfaces" section describes how to insert data separately from the query when using the command-line client or the HTTP interface. - -The query may optionally specify a list of columns for insertion. In this case, the default values are written to the other columns. -Default values are calculated from DEFAULT expressions specified in table definitions, or, if the DEFAULT is not explicitly defined, zeros and empty strings are used. If the 'strict_insert_default' setting is set to 1, all the columns that do not have explicit DEFAULTS must be specified in the query. - -.. code-block:: sql - - INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - -Inserts the result of the SELECT query into a table. -The names and data types of the SELECT result must exactly match the table structure that data is inserted into, or the specified list of columns. -To change column names, use synonyms (AS) in the SELECT query. -To change data types, use type conversion functions (see the section "Functions"). - -None of the data formats allows using expressions as values. -In other words, you can't write INSERT INTO t VALUES (now(), 1 + 1, DEFAULT). - -There is no support for other data part modification queries: -UPDATE, DELETE, REPLACE, MERGE, UPSERT, INSERT UPDATE. -However, you can delete old data using ALTER TABLE ... DROP PARTITION. - - -SELECT -~~~~~~ - -His Highness, the SELECT query. - -.. code-block:: sql - - SELECT [DISTINCT] expr_list - [FROM [db.]table | (subquery) | table_function] [FINAL] - [SAMPLE sample_coeff] - [ARRAY JOIN ...] - [GLOBAL] ANY|ALL INNER|LEFT JOIN (subquery)|table USING columns_list - [PREWHERE expr] - [WHERE expr] - [GROUP BY expr_list] [WITH TOTALS] - [HAVING expr] - [ORDER BY expr_list] - [LIMIT [n, ]m] - [UNION ALL ...] - [INTO OUTFILE filename] - [FORMAT format] - -All the clauses are optional, except for the required list of expressions immediately after SELECT. -The clauses below are described in almost the same order as in the query execution conveyor. - -If the query omits the DISTINCT, GROUP BY, and ORDER BY clauses and the IN and JOIN subqueries, the query will be completely stream processed, using O(1) amount of RAM. -Otherwise, the query may consume too much RAM, if appropriate restrictions are not defined (max_memory_usage, max_rows_to_group_by, max_rows_to_sort, max_rows_in_distinct, max_bytes_in_distinct, max_rows_in_set, max_bytes_in_set, max_rows_in_join, max_bytes_in_join, max_bytes_before_external_sort, max_bytes_before_external_group_by). For more information, see the section "Settings". It is possible to use external sorting (saving temporary tables to a disk) and external aggregation. Merge join is not implemented. - -FROM clause -""""""""""" - -If the FROM clause is omitted, data will be read from the 'system.one' table. -The 'system.one' table contains exactly one row (this table fulfills the same purpose as the DUAL table found in other DBMSs). - -The FROM clause specifies the table to read data from, or a subquery, or a table function; ARRAY JOIN and the regular JOIN may also be included (see below). - -Instead of a table, the SELECT subquery may be specified in brackets. In this case, the subquery processing pipeline will be built into the processing pipeline of an external query. -In contrast to standard SQL, a synonym does not need to be specified after a subquery. For compatibility, it is possible to write 'AS name' after a subquery, but the specified name isn't used anywhere. - -A table function may be specified instead of a table. For more information, see the section "Table functions". - -To execute a query, all the columns listed in the query are extracted from the appropriate table. Any columns not needed for the external query are thrown out of the subqueries. -If a query does not list any columns (for example, SELECT count() FROM t), some column is extracted from the table anyway (the smallest one is preferred), in order to calculate the number of rows. - -The FINAL modifier can be used only for a SELECT from a CollapsingMergeTree table. When you specify FINAL, data is selected fully "collapsed". Keep in mind that using FINAL leads to a selection that includes columns related to the primary key, in addition to the columns specified in the SELECT. Additionally, the query will be executed in a single stream, and data will be merged during query execution. This means that when using FINAL, the query is processed more slowly. In most cases, you should avoid using FINAL. For more information, see the section "CollapsingMergeTree engine". - -SAMPLE clause -""""""""""""" - -The SAMPLE clause allows for approximated query processing. -Approximated query processing is only supported by MergeTree\* type tables, and only if the sampling expression was specified during table creation (see the section "MergeTree engine"). - -SAMPLE has the format ``SAMPLE k``, where 'k' is a decimal number from 0 to 1, or ``SAMPLE n``, where 'n' is a sufficiently large integer. - -In the first case, the query will be executed on 'k' percent of data. For example, ``SAMPLE 0.1`` runs the query on 10% of data. -In the second case, the query will be executed on a sample of no more than 'n' rows. For example, ``SAMPLE 10000000`` runs the query on a maximum of 10,000,000 rows. - -Example: - -.. code-block:: sql - - SELECT - Title, - count() * 10 AS PageViews - FROM hits_distributed - SAMPLE 0.1 - WHERE - CounterID = 34 - AND toDate(EventDate) >= toDate('2013-01-29') - AND toDate(EventDate) <= toDate('2013-02-04') - AND NOT DontCountHits - AND NOT Refresh - AND Title != '' - GROUP BY Title - ORDER BY PageViews DESC LIMIT 1000 - -In this example, the query is executed on a sample from 0.1 (10%) of data. Values of aggregate functions are not corrected automatically, so to get an approximate result, the value 'count()' is manually multiplied by 10. - -When using something like ``SAMPLE 10000000``, there isn't any information about which relative percent of data was processed or what the aggregate functions should be multiplied by, so this method of writing is not always appropriate to the situation. - -A sample with a relative coefficient is "consistent": if we look at all possible data that could be in the table, a sample (when using a single sampling expression specified during table creation) with the same coefficient always selects the same subset of possible data. In other words, a sample from different tables on different servers at different times is made the same way. - -For example, a sample of user IDs takes rows with the same subset of all the possible user IDs from different tables. This allows using the sample in subqueries in the IN clause, as well as for manually correlating results of different queries with samples. - -ARRAY JOIN clause -""""""""""""""""" - -Allows executing JOIN with an array or nested data structure. The intent is similar to the 'arrayJoin' function, but its functionality is broader. - -ARRAY JOIN is essentially INNER JOIN with an array. Example: - -.. code-block:: text - - :) CREATE TABLE arrays_test (s String, arr Array(UInt8)) ENGINE = Memory - - CREATE TABLE arrays_test - ( - s String, - arr Array(UInt8) - ) ENGINE = Memory - - Ok. - - 0 rows in set. Elapsed: 0.001 sec. - - :) INSERT INTO arrays_test VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []) - - INSERT INTO arrays_test VALUES - - Ok. - - 3 rows in set. Elapsed: 0.001 sec. - - :) SELECT * FROM arrays_test - - SELECT * - FROM arrays_test - - ┌─s───────┬─arr─────┐ - │ Hello │ [1,2] │ - │ World │ [3,4,5] │ - │ Goodbye │ [] │ - └─────────┴─────────┘ - - 3 rows in set. Elapsed: 0.001 sec. - - :) SELECT s, arr FROM arrays_test ARRAY JOIN arr - - SELECT s, arr - FROM arrays_test - ARRAY JOIN arr - - ┌─s─────┬─arr─┐ - │ Hello │ 1 │ - │ Hello │ 2 │ - │ World │ 3 │ - │ World │ 4 │ - │ World │ 5 │ - └───────┴─────┘ - - 5 rows in set. Elapsed: 0.001 sec. - -An alias can be specified for an array in the ARRAY JOIN clause. In this case, an array item can be accessed by this alias, but the array itself by the original name. Example: - -.. code-block:: sql - - :) SELECT s, arr, a FROM arrays_test ARRAY JOIN arr AS a - - SELECT s, arr, a - FROM arrays_test - ARRAY JOIN arr AS a - -.. code-block:: text - - ┌─s─────┬─arr─────┬─a─┐ - │ Hello │ [1,2] │ 1 │ - │ Hello │ [1,2] │ 2 │ - │ World │ [3,4,5] │ 3 │ - │ World │ [3,4,5] │ 4 │ - │ World │ [3,4,5] │ 5 │ - └───────┴─────────┴───┘ - - 5 rows in set. Elapsed: 0.001 sec. - -Multiple arrays of the same size can be comma-separated in the ARRAY JOIN clause. In this case, JOIN is performed with them simultaneously (the direct sum, not the direct product). -Example: - -.. code-block:: text - - :) SELECT s, arr, a, num, mapped FROM arrays_test ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(x -> x + 1, arr) AS mapped - - SELECT s, arr, a, num, mapped - FROM arrays_test - ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num, arrayMap(lambda(tuple(x), plus(x, 1)), arr) AS mapped - - ┌─s─────┬─arr─────┬─a─┬─num─┬─mapped─┐ - │ Hello │ [1,2] │ 1 │ 1 │ 2 │ - │ Hello │ [1,2] │ 2 │ 2 │ 3 │ - │ World │ [3,4,5] │ 3 │ 1 │ 4 │ - │ World │ [3,4,5] │ 4 │ 2 │ 5 │ - │ World │ [3,4,5] │ 5 │ 3 │ 6 │ - └───────┴─────────┴───┴─────┴────────┘ - - 5 rows in set. Elapsed: 0.002 sec. - - :) SELECT s, arr, a, num, arrayEnumerate(arr) FROM arrays_test ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num - - SELECT s, arr, a, num, arrayEnumerate(arr) - FROM arrays_test - ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num - - ┌─s─────┬─arr─────┬─a─┬─num─┬─arrayEnumerate(arr)─┐ - │ Hello │ [1,2] │ 1 │ 1 │ [1,2] │ - │ Hello │ [1,2] │ 2 │ 2 │ [1,2] │ - │ World │ [3,4,5] │ 3 │ 1 │ [1,2,3] │ - │ World │ [3,4,5] │ 4 │ 2 │ [1,2,3] │ - │ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ - └───────┴─────────┴───┴─────┴─────────────────────┘ - - 5 rows in set. Elapsed: 0.002 sec. - -ARRAY JOIN also works with nested data structures. Example: - -.. code-block:: text - - :) CREATE TABLE nested_test (s String, nest Nested(x UInt8, y UInt32)) ENGINE = Memory - - CREATE TABLE nested_test - ( - s String, - nest Nested( - x UInt8, - y UInt32) - ) ENGINE = Memory - - Ok. - - 0 rows in set. Elapsed: 0.006 sec. - - :) INSERT INTO nested_test VALUES ('Hello', [1,2], [10,20]), ('World', [3,4,5], [30,40,50]), ('Goodbye', [], []) - - INSERT INTO nested_test VALUES - - Ok. - - 3 rows in set. Elapsed: 0.001 sec. - - :) SELECT * FROM nested_test - - SELECT * - FROM nested_test - - ┌─s───────┬─nest.x──┬─nest.y─────┐ - │ Hello │ [1,2] │ [10,20] │ - │ World │ [3,4,5] │ [30,40,50] │ - │ Goodbye │ [] │ [] │ - └─────────┴─────────┴────────────┘ - - 3 rows in set. Elapsed: 0.001 sec. - - :) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest - - SELECT s, `nest.x`, `nest.y` - FROM nested_test - ARRAY JOIN nest - - ┌─s─────┬─nest.x─┬─nest.y─┐ - │ Hello │ 1 │ 10 │ - │ Hello │ 2 │ 20 │ - │ World │ 3 │ 30 │ - │ World │ 4 │ 40 │ - │ World │ 5 │ 50 │ - └───────┴────────┴────────┘ - - 5 rows in set. Elapsed: 0.001 sec. - -When specifying names of nested data structures in ARRAY JOIN, the meaning is the same as ARRAY JOIN with all the array elements that it consists of. Example: - -.. code-block:: text - - :) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest.x, nest.y - - SELECT s, `nest.x`, `nest.y` - FROM nested_test - ARRAY JOIN `nest.x`, `nest.y` - - ┌─s─────┬─nest.x─┬─nest.y─┐ - │ Hello │ 1 │ 10 │ - │ Hello │ 2 │ 20 │ - │ World │ 3 │ 30 │ - │ World │ 4 │ 40 │ - │ World │ 5 │ 50 │ - └───────┴────────┴────────┘ - - 5 rows in set. Elapsed: 0.001 sec. - -This variation also makes sense: - -.. code-block:: sql - - :) SELECT s, nest.x, nest.y FROM nested_test ARRAY JOIN nest.x - - SELECT s, `nest.x`, `nest.y` - FROM nested_test - ARRAY JOIN `nest.x` - -.. code-block:: text - - ┌─s─────┬─nest.x─┬─nest.y─────┐ - │ Hello │ 1 │ [10,20] │ - │ Hello │ 2 │ [10,20] │ - │ World │ 3 │ [30,40,50] │ - │ World │ 4 │ [30,40,50] │ - │ World │ 5 │ [30,40,50] │ - └───────┴────────┴────────────┘ - - 5 rows in set. Elapsed: 0.001 sec. - -An alias may be used for a nested data structure, in order to select either the JOIN result or the source array. Example: - -.. code-block:: sql - - :) SELECT s, n.x, n.y, nest.x, nest.y FROM nested_test ARRAY JOIN nest AS n - - SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y` - FROM nested_test - ARRAY JOIN nest AS n - -.. code-block:: text - - ┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┐ - │ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ - │ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ - │ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ - │ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ - │ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ - └───────┴─────┴─────┴─────────┴────────────┘ - - 5 rows in set. Elapsed: 0.001 sec. - -Example of using the arrayEnumerate function: - -.. code-block:: sql - - :) SELECT s, n.x, n.y, nest.x, nest.y, num FROM nested_test ARRAY JOIN nest AS n, arrayEnumerate(nest.x) AS num - - SELECT s, `n.x`, `n.y`, `nest.x`, `nest.y`, num - FROM nested_test - ARRAY JOIN nest AS n, arrayEnumerate(`nest.x`) AS num - -.. code-block:: text - - ┌─s─────┬─n.x─┬─n.y─┬─nest.x──┬─nest.y─────┬─num─┐ - │ Hello │ 1 │ 10 │ [1,2] │ [10,20] │ 1 │ - │ Hello │ 2 │ 20 │ [1,2] │ [10,20] │ 2 │ - │ World │ 3 │ 30 │ [3,4,5] │ [30,40,50] │ 1 │ - │ World │ 4 │ 40 │ [3,4,5] │ [30,40,50] │ 2 │ - │ World │ 5 │ 50 │ [3,4,5] │ [30,40,50] │ 3 │ - └───────┴─────┴─────┴─────────┴────────────┴─────┘ - - 5 rows in set. Elapsed: 0.002 sec. - -The query can only specify a single ARRAY JOIN clause. - -The corresponding conversion can be performed before the WHERE/PREWHERE clause (if its result is needed in this clause), or after completing WHERE/PREWHERE (to reduce the volume of calculations). - -JOIN clause -""""""""""" -The normal JOIN, which is not related to ARRAY JOIN described above. - -.. code-block:: sql - - [GLOBAL] ANY|ALL INNER|LEFT [OUTER] JOIN (subquery)|table USING columns_list - -Performs joins with data from the subquery. At the beginning of query execution, the subquery specified after JOIN is run, and its result is saved in memory. Then it is read from the "left" table specified in the FROM clause, and while it is being read, for each of the read rows from the "left" table, rows are selected from the subquery results table (the "right" table) that meet the condition for matching the values of the columns specified in USING. - -The table name can be specified instead of a subquery. This is equivalent to the ``SELECT * FROM table`` subquery, except in a special case when the table has the Join engine - an array prepared for joining. - -All columns that are not needed for the JOIN are deleted from the subquery. - -There are several types of JOINs: - -INNER or LEFT - the type: -If INNER is specified, the result will contain only those rows that have a matching row in the right table. -If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything. - -ANY or ALL - strictness: -If ANY is specified and there are multiple matching rows in the right table, only the first one will be joined. -If ALL is specified and there are multiple matching rows in the right table, the data will be multiplied by the number of these rows. - -Using ALL corresponds to the normal JOIN semantic from standard SQL. -Using ANY is optimal. If the right table has only one matching row, the results of ANY and ALL are the same. You must specify either ANY or ALL (neither of them is selected by default). - -GLOBAL - distribution: - -When using a normal ``JOIN``, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. - -When using ``GLOBAL ... JOIN``, first the requestor server runs a subquery to calculate the right table. This temporary table is passed to each remote server, and queries are run on them using the temporary data that was transmitted. - -Be careful when using GLOBAL JOINs. For more information, see the section "Distributed subqueries" below. - -Any combination of JOINs is possible. For example, ``GLOBAL ANY LEFT OUTER JOIN``. - -When running JOINs, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in WHERE and before aggregation. In order to explicitly set the order of execution, we recommend running a JOIN subquery with a subquery. - -Example: - -.. code-block:: sql - - SELECT - CounterID, - hits, - visits - FROM - ( - SELECT - CounterID, - count() AS hits - FROM test.hits - GROUP BY CounterID - ) ANY LEFT JOIN - ( - SELECT - CounterID, - sum(Sign) AS visits - FROM test.visits - GROUP BY CounterID - ) USING CounterID - ORDER BY hits DESC - LIMIT 10 - -.. code-block:: text - - ┌─CounterID─┬───hits─┬─visits─┐ - │ 1143050 │ 523264 │ 13665 │ - │ 731962 │ 475698 │ 102716 │ - │ 722545 │ 337212 │ 108187 │ - │ 722889 │ 252197 │ 10547 │ - │ 2237260 │ 196036 │ 9522 │ - │ 23057320 │ 147211 │ 7689 │ - │ 722818 │ 90109 │ 17847 │ - │ 48221 │ 85379 │ 4652 │ - │ 19762435 │ 77807 │ 7026 │ - │ 722884 │ 77492 │ 11056 │ - └───────────┴────────┴────────┘ - -Subqueries don't allow you to set names or use them for referencing a column from a specific subquery. -The columns specified in USING must have the same names in both subqueries, and the other columns must be named differently. You can use aliases to change the names of columns in subqueries (the example uses the aliases 'hits' and 'visits'). - -The USING clause specifies one or more columns to join, which establishes the equality of these columns. The list of columns is set without brackets. More complex join conditions are not supported. - -The right table (the subquery result) resides in RAM. If there isn't enough memory, you can't run a JOIN. - -Only one JOIN can be specified in a query (on a single level). To run multiple JOINs, you can put them in subqueries. - -Each time a query is run with the same JOIN, the subquery is run again - the result is not cached. To avoid this, use the special 'Join' table engine, which is a prepared array for joining that is always in RAM. For more information, see the section "Table engines, Join". - -In some cases, it is more efficient to use IN instead of JOIN. Among the various types of JOINs, the most efficient is ANY LEFT JOIN, then ANY INNER JOIN. The least efficient are ALL LEFT JOIN and ALL INNER JOIN. - -If you need a JOIN for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a JOIN might not be very convenient due to the bulky syntax and the fact that the right table is re-accessed for every query. For such cases, there is an "external dictionaries" feature that you should use instead of JOIN. For more information, see the section "External dictionaries". - -WHERE clause -"""""""""""" - -If there is a WHERE clause, it must contain an expression with the UInt8 type. This is usually an expression with comparison and logical operators. -This expression will be used for filtering data before all other transformations. - -If indexes are supported by the database table engine, the expression is evaluated on the ability to use indexes. - -PREWHERE clause -""""""""""""""" - -This clause has the same meaning as the WHERE clause. The difference is in which data is read from the table. When using PREWHERE, first only the columns necessary for executing PREWHERE are read. Then the other columns are read that are needed for running the query, but only those blocks where the PREWHERE expression is true. - -It makes sense to use PREWHERE if there are filtration conditions that are not suitable for indexes that are used by a minority of the columns in the query, but that provide strong data filtration. This reduces the volume of data to read. - -For example, it is useful to write PREWHERE for queries that extract a large number of columns, but that only have filtration for a few columns. - -PREWHERE is only supported by \*MergeTree tables. - -A query may simultaneously specify PREWHERE and WHERE. In this case, PREWHERE precedes WHERE. - -Keep in mind that it does not make much sense for PREWHERE to only specify those columns that have an index, because when using an index, only the data blocks that match the index are read. - -If the 'optimize_move_to_prewhere' setting is set to 1 and PREWHERE is omitted, the system uses heuristics to automatically move parts of expressions from WHERE to PREWHERE. - -GROUP BY clause -""""""""""""""" - -This is one of the most important parts of a column-oriented DBMS. - -If there is a GROUP BY clause, it must contain a list of expressions. Each expression will be referred to here as a "key". -All the expressions in the SELECT, HAVING, and ORDER BY clauses must be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. - -If a query contains only table columns inside aggregate functions, the GROUP BY clause can be omitted, and aggregation by an empty set of keys is assumed. - -Example: - -.. code-block:: sql - - SELECT - count(), - median(FetchTiming > 60 ? 60 : FetchTiming), - count() - sum(Refresh) - FROM hits - -However, in contrast to standard SQL, if the table doesn't have any rows (either there aren't any at all, or there aren't any after using WHERE to filter), an empty result is returned, and not the result from one of the rows containing the initial values of aggregate functions. - -As opposed to MySQL (and conforming to standard SQL), you can't get some value of some column that is not in a key or aggregate function (except constant expressions). To work around this, you can use the 'any' aggregate function (get the first encountered value) or 'min/max'. - -Example: - -.. code-block:: sql - - SELECT - domainWithoutWWW(URL) AS domain, - count(), - any(Title) AS title -- для каждого домена достаём первый попавшийся заголовок страницы - FROM hits - GROUP BY domain - -For every different key value encountered, GROUP BY calculates a set of aggregate function values. - -GROUP BY is not supported for array columns. - -A constant can't be specified as arguments for aggregate functions. Example: sum(1). Instead of this, you can get rid of the constant. Example: ``count()``. - -WITH TOTALS modifier -^^^^^^^^^^^^^^^^^^^^ - -If the WITH TOTALS modifier is specified, another row will be calculated. This row will have key columns containing default values (zeros or empty lines), and columns of aggregate functions with the values calculated across all the rows (the "total" values). - -This extra row is output in JSON*, TabSeparated*, and Pretty* formats, separately from the other rows. In the other formats, this row is not output. - -In JSON* formats, this row is output as a separate 'totals' field. In TabSeparated formats, the row comes after the main result, preceded by an empty row (after the other data). In Pretty formats, the row is output as a separate table after the main result. - -``WITH TOTALS`` can be run in different ways when HAVING is present. The behavior depends on the 'totals_mode' setting. -By default, totals_mode = 'before_having'. In this case, 'totals' is calculated across all rows, including the ones that don't pass through HAVING and 'max_rows_to_group_by'. - -The other alternatives include only the rows that pass through HAVING in 'totals', and behave differently with the setting 'max_rows_to_group_by' and 'group_by_overflow_mode = 'any''. - -``after_having_exclusive`` - Don't include rows that didn't pass through ``'max_rows_to_group_by'``. In other words, 'totals' will have less than or the same number of rows as it would if 'max_rows_to_group_by' were omitted. - -``after_having_inclusive`` - Include all the rows that didn't pass through ``'max_rows_to_group_by'`` in 'totals'. In other words, 'totals' will have more than or the same number of rows as it would if 'max_rows_to_group_by' were omitted. - -``after_having_auto`` - Count the number of rows that passed through HAVING. If it is more than a certain amount (by default, 50%), include all the rows that didn't pass through 'max_rows_to_group_by' in 'totals'. Otherwise, do not include them. - -``totals_auto_threshold`` - By default, 0.5 is the coefficient for ``after_having_auto``. - -If 'max_rows_to_group_by' and 'group_by_overflow_mode = 'any'' are not used, all variations of 'after_having' are the same, and you can use any of them (for example, 'after_having_auto'). - -You can use WITH TOTALS in subqueries, including subqueries in the JOIN clause. In this case, the respective total values are combined. - -external memory GROUP BY -^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to turn on spilling temporary data to disk to limit memory consumption during the execution of GROUP BY. Value of ``max_bytes_before_external_group_by`` setting determines the maximum memory consumption before temporary data is dumped to the file system. If it is 0 (the default value), the feature is turned off. - -When using ``max_bytes_before_external_group_by`` it is advisable to set ``max_memory_usage`` to an approximately twice greater value. The reason for this is that aggregation is executed in two stages: reading and generation of intermediate data (1) and merging of intermediate data (2). Spilling data to the filesystem can be performed only on stage 1. If the spilling did not happen, then stage 2 could consume up to the same amount of memory as stage 1. - -For example: if ``max_memory_usage`` is equal to 10000000000 and you want to use external aggregation, it makes sense to set ``max_bytes_before_external_group_by`` to 10000000000 and ``max_memory_usage`` to 20000000000. If dumping data to the file system happened at least once during the execution, maximum memory consumption would be just a little bit higher than ``max_bytes_before_external_group_by``. - -During distributed query execution external aggregation is performed on the remote servers. If you want the memory consumption on the originating server to be small, set ``distributed_aggregation_memory_efficient`` to 1. If ``distributed_aggregation_memory_efficient`` is turned on then during merging of the dumped data and also during merging of the query results from the remote servers, total memory consumption is no more than 1/256 * number of threads of the total amount of memory. - -If external aggregation is turned on and total memory consumption was less than ``max_bytes_before_external_group_by`` (meaning that no spilling took place), the query performance is the same as when external aggregation is turned off. If some data was dumped, then execution time will be several times longer (approximately 3x). - -If you have an ORDER BY clause with some small LIMIT after a GROUP BY, then ORDER BY will not consume significant amount of memory. But if no LIMIT is provided, don't forget to turn on external sorting (``max_bytes_before_external_sort``). - -LIMIT N BY modifier -^^^^^^^^^^^^^^^^^^^ - -LIMIT ``N`` BY ``COLUMNS`` allows you to restrict top ``N`` rows per each group of ``COLUMNS``. ``LIMIT N BY`` is unrelated to ``LIMIT`` clause. Key for ``LIMIT N BY`` could contain arbitrary number of columns or expressions. - -Example: - -.. code-block:: sql - - SELECT - domainWithoutWWW(URL) AS domain, - domainWithoutWWW(REFERRER_URL) AS referrer, - device_type, - count() cnt - FROM hits - GROUP BY domain, referrer, device_type - ORDER BY cnt DESC - LIMIT 5 BY domain, device_type - LIMIT 100 - -will select top 5 referrers for each domain - device type pair, total number of rows - 100. - -HAVING clause -""""""""""""" - -Allows filtering the result received after GROUP BY, similar to the WHERE clause. -WHERE and HAVING differ in that WHERE is performed before aggregation (GROUP BY), while HAVING is performed after it. If aggregation is not performed, HAVING can't be used. - -ORDER BY clause -""""""""""""""" - -The ORDER BY clause contains a list of expressions, which can each be assigned DESC or ASC (the sorting direction). If the direction is not specified, ASC is assumed. ASC is sorted in ascending order, and DESC in descending order. The sorting direction applies to a single expression, not to the entire list. Example: ``ORDER BY Visits DESC, SearchPhrase`` - -For sorting by String values, you can specify collation (comparison). Example: ``ORDER BY SearchPhrase COLLATE 'tr'`` - for sorting by keyword in ascending order, using the Turkish alphabet, case insensitive, assuming that strings are UTF-8 encoded. COLLATE can be specified or not for each expression in ORDER BY independently. If ASC or DESC is specified, COLLATE is specified after it. When using COLLATE, sorting is always case-insensitive. - -We only recommend using COLLATE for final sorting of a small number of rows, since sorting with COLLATE is less efficient than normal sorting by bytes. - -Rows that have identical values for the list of sorting expressions are output in an arbitrary order, which can also be nondeterministic (different each time). -If the ORDER BY clause is omitted, the order of the rows is also undefined, and may be nondeterministic as well. - -When floating point numbers are sorted, NaNs are separate from the other values. Regardless of the sorting order, NaNs come at the end. In other words, for ascending sorting they are placed as if they are larger than all the other numbers, while for descending sorting they are placed as if they are smaller than the rest. - -Less RAM is used if a small enough LIMIT is specified in addition to ORDER BY. Otherwise, the amount of memory spent is proportional to the volume of data for sorting. For distributed query processing, if GROUP BY is omitted, sorting is partially done on remote servers, and the results are merged on the requestor server. This means that for distributed sorting, the volume of data to sort can be greater than the amount of memory on a single server. - -If there is not enough RAM, it is possible to perform sorting in external memory (creating temporary files on a disk). Use the setting max_bytes_before_external_sort for this purpose. If it is set to 0 (the default), external sorting is disabled. If it is enabled, when the volume of data to sort reaches the specified number of bytes, the collected data is sorted and dumped into a temporary file. After all data is read, all the sorted files are merged and the results are output. Files are written to the /var/lib/clickhouse/tmp/ directory in the config (by default, but you can use the 'tmp_path' parameter to change this setting). - -Running a query may use more memory than ``'max_bytes_before_external_sort'``. For this reason, this setting must have a value significantly smaller than 'max_memory_usage'. As an example, if your server has 128 GB of RAM and you need to run a single query, set 'max_memory_usage' to 100 GB, and 'max_bytes_before_external_sort' to 80 GB. - -External sorting works much less effectively than sorting in RAM. - -SELECT clause -""""""""""""" - -The expressions specified in the SELECT clause are analyzed after the calculations for all the clauses listed above are completed. -More specifically, expressions are analyzed that are above the aggregate functions, if there are any aggregate functions. The aggregate functions and everything below them are calculated during aggregation (GROUP BY). These expressions work as if they are applied to separate rows in the result. - -DISTINCT clause -""""""""""""""" - -If DISTINCT is specified, only a single row will remain out of all the sets of fully matching rows in the result. -The result will be the same as if GROUP BY were specified across all the fields specified in SELECT without aggregate functions. But there are several differences from GROUP BY: - -- DISTINCT can be applied together with GROUP BY. -- When ORDER BY is omitted and LIMIT is defined, the query stops running immediately after the required number of different rows has been read. In this case, using DISTINCT is much more optimal. -- Data blocks are output as they are processed, without waiting for the entire query to finish running. - -DISTINCT is not supported if SELECT has at least one array column. - -LIMIT clause -"""""""""""" - -LIMIT m allows you to select the first 'm' rows from the result. -LIMIT n, m allows you to select the first 'm' rows from the result after skipping the first 'n' rows. - -'n' and 'm' must be non-negative integers. - -If there isn't an ORDER BY clause that explicitly sorts results, the result may be arbitrary and nondeterministic. - -UNION ALL clause -"""""""""""""""" - -You can use UNION ALL to combine any number of queries. Example: - -.. code-block:: sql - - SELECT CounterID, 1 AS table, toInt64(count()) AS c - FROM test.hits - GROUP BY CounterID - - UNION ALL - - SELECT CounterID, 2 AS table, sum(Sign) AS c - FROM test.visits - GROUP BY CounterID - HAVING c > 0 - -Only UNION ALL is supported. The regular UNION (UNION DISTINCT) is not supported. If you need UNION DISTINCT, you can write SELECT DISTINCT from a subquery containing UNION ALL. - -Queries that are parts of UNION ALL can be run simultaneously, and their results can be mixed together. - -The structure of results (the number and type of columns) must match for the queries, but the column names can differ. In this case, the column names for the final result will be taken from the first query. - -Queries that are parts of UNION ALL can't be enclosed in brackets. ORDER BY and LIMIT are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with UNION ALL in a subquery in the FROM clause. - -INTO OUTFILE clause -""""""""""""""""""" - -Add ``INTO OUTFILE`` filename clause (where filename is a string literal) to redirect query output to a file filename. -In contrast to MySQL the file is created on a client host. The query will fail if a file with the same filename already exists. -INTO OUTFILE is available in the command-line client and clickhouse-local (a query sent via HTTP interface will fail). - -Default output format is TabSeparated (the same as in the batch mode of command-line client). - -FORMAT clause -""""""""""""" -Specify 'FORMAT format' to get data in any specified format. -You can use this for convenience, or for creating dumps. For more information, see the section "Formats". -If the FORMAT clause is omitted, the default format is used, which depends on both the settings and the interface used for accessing the DB. For the HTTP interface and the command-line client in batch mode, the default format is TabSeparated. For the command-line client in interactive mode, the default format is PrettyCompact (it has attractive and compact tables). - -When using the command-line client, data is passed to the client in an internal efficient format. The client independently interprets the FORMAT clause of the query and formats the data itself (thus relieving the network and the server from the load). - -IN operators -"""""""""""" - -The ``IN``, ``NOT IN``, ``GLOBAL IN``, and ``GLOBAL NOT IN`` operators are covered separately, since their functionality is quite rich. - -The left side of the operator is either a single column or a tuple. - -Examples: - -.. code-block:: sql - - SELECT UserID IN (123, 456) FROM ... - SELECT (CounterID, UserID) IN ((34, 123), (101500, 456)) FROM ... - -If the left side is a single column that is in the index, and the right side is a set of constants, the system uses the index for processing the query. - -Don't list too many values explicitly (i.e. millions). If a data set is large, put it in a temporary table (for example, see the section "External data for query processing"), then use a subquery. - -The right side of the operator can be a set of constant expressions, a set of tuples with constant expressions (shown in the examples above), or the name of a database table or SELECT subquery in brackets. - -If the right side of the operator is the name of a table (for example, ``UserID IN users``), this is equivalent to the subquery ``UserID IN (SELECT * FROM users)``. Use this when working with external data that is sent along with the query. For example, the query can be sent together with a set of user IDs loaded to the 'users' temporary table, which should be filtered. - -If the right side of the operator is a table name that has the Set engine (a prepared data set that is always in RAM), the data set will not be created over again for each query. - -The subquery may specify more than one column for filtering tuples. -Example: - -.. code-block:: sql - - SELECT (CounterID, UserID) IN (SELECT CounterID, UserID FROM ...) FROM ... - -The columns to the left and right of the ``IN`` operator should have the same type. - -The IN operator and subquery may occur in any part of the query, including in aggregate functions and lambda functions. -Example: - -.. code-block:: sql - - SELECT - EventDate, - avg(UserID IN - ( - SELECT UserID - FROM test.hits - WHERE EventDate = toDate('2014-03-17') - )) AS ratio - FROM test.hits - GROUP BY EventDate - ORDER BY EventDate ASC - -.. code-block:: text - - ┌──EventDate─┬────ratio─┐ - │ 2014-03-17 │ 1 │ - │ 2014-03-18 │ 0.807696 │ - │ 2014-03-19 │ 0.755406 │ - │ 2014-03-20 │ 0.723218 │ - │ 2014-03-21 │ 0.697021 │ - │ 2014-03-22 │ 0.647851 │ - │ 2014-03-23 │ 0.648416 │ - └────────────┴──────────┘ - -For each day after March 17th, count the percentage of pageviews made by users who visited the site on March 17th. -A subquery in the IN clause is always run just one time on a single server. There are no dependent subqueries. - -Distributed subqueries -"""""""""""""""""""""" - -There are two versions of INs with subqueries (and for JOINs): the regular ``IN`` / ``JOIN``, and ``GLOBAL IN`` / ``GLOBAL JOIN``. They differ in how they are run for distributed query processing. - -When using the regular ``IN``, the query is sent to remote servers, and each of them runs the subqueries in the IN or JOIN clause. - -When using ``GLOBAL IN`` / ``GLOBAL JOIN``, first all the subqueries for ``GLOBAL IN`` / ``GLOBAL JOIN`` are run, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. - -For a non-distributed query, use the regular ``IN`` / ``JOIN``. - -Be careful when using subqueries in the ``IN`` / ``JOIN`` clauses for distributed query processing. - -Let's look at some examples. Assume that each server in the cluster has a normal local_table. Each server also has a **distributed_table** table with the Distributed type, which looks at all the servers in the cluster. - -For a query to the **distributed_table**, the query will be sent to all the remote servers and run on them using the **local_table**. - -For example, the query - -``SELECT uniq(UserID) FROM distributed_table`` - -will be sent to all the remote servers as - -``SELECT uniq(UserID) FROM local_table`` - -and run on each of them in parallel, until it reaches the stage where intermediate results can be combined. Then the intermediate results will be returned to the requestor server and merged on it, and the final result will be sent to the client. - -Now let's examine a query with IN: - -.. code-block:: sql - - SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) - -- calculates the overlap in the audiences of two websites. - -This query will be sent to all the remote servers as - -.. code-block:: sql - - SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM local_table WHERE CounterID = 34) - -In other words, the data set in the IN clause will be collected on each server independently, only across the data that is stored locally on each of the servers. - -This will work correctly and optimally if you are prepared for this case and have spread data across the cluster servers such that the data for a single UserID resides entirely on a single server. In this case, all the necessary data will be available locally on each server. Otherwise, the result will be inaccurate. We refer to this variation of the query as "local IN". - -To correct how the query works when data is spread randomly across the cluster servers, you could specify **distributed_table** inside a subquery. The query would look like this: - -.. code-block:: sql - - SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) - -This query will be sent to all remote servers as - -.. code-block:: sql - - SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) - -Each of the remote servers will start running the subquery. Since the subquery uses a distributed table, each remote server will re-send the subquery to every remote server, as - -.. code-block:: sql - - SELECT UserID FROM local_table WHERE CounterID = 34 - -For example, if you have a cluster of 100 servers, executing the entire query will require 10,000 elementary requests, which is generally considered unacceptable. - -In such cases, you should always use ``GLOBAL IN`` instead of ``IN``. Let's look at how it works for the query - -.. code-block:: sql - - SELECT uniq(UserID) FROM distributed_table WHERE CounterID = 101500 AND UserID GLOBAL IN (SELECT UserID FROM distributed_table WHERE CounterID = 34) - -The requestor server will execute the subquery - -.. code-block:: sql - - SELECT UserID FROM distributed_table WHERE CounterID = 34 - -and the result will be put in a temporary table in RAM. Then a query will be sent to each remote server as - -.. code-block:: sql - - SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL IN _data1 - -and the temporary table '_data1' will be sent to every remote server together with the query (the name of the temporary table is implementation-defined). - -This is more optimal than using the normal IN. However, keep the following points in mind: - -#. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don't need to do this for a normal IN.) -#. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. -#. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. -#. Try to distribute data across servers so that you don't need to use GLOBAL IN on a regular basis. -#. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that in each datacenter, there will be at least one replica of each shard, and there is a fast network between them - for possibility to process query with transferring data only inside datacenter. - -It also makes sense to specify a local table in the GLOBAL IN clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. - -Extreme values -"""""""""""""" - -In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the 'extremes' setting to '1'. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. - -An extra two rows are calculated - the minimums and maximums, respectively. These extra two rows are output in JSON*, TabSeparated*, and Pretty* formats, separate from the other rows. They are not output for other formats. - -In JSON* formats, the extreme values are output in a separate 'extremes' field. In TabSeparated formats, the row comes after the main result, and after 'totals' if present. It is preceded by an empty row (after the other data). In Pretty formats, the row is output as a separate table after the main result, and after 'totals' if present. - -Extreme values are calculated for rows that have passed through LIMIT. However, when using 'LIMIT offset, size', the rows before 'offset' are included in 'extremes'. In stream requests, the result may also include a small number of rows that passed through LIMIT. - -Notes -""""" - -The GROUP BY and ORDER BY clauses do not support positional arguments. This contradicts MySQL, but conforms to standard SQL. -For example, ``'GROUP BY 1, 2'`` will be interpreted as grouping by constants (i.e. aggregation of all rows into one). - -You can use synonyms (AS aliases) in any part of a query. - -You can put an asterisk in any part of a query instead of an expression. When the query is analyzed, the asterisk is expanded to a list of all table columns (excluding the ``MATERIALIZED`` and ALIAS columns). There are only a few cases when using an asterisk is justified: -* When creating a table dump. -* For tables containing just a few columns, such as system tables. -* For getting information about what columns are in a table. In this case, set ``'LIMIT 1'``. But it is better to use the ``DESC TABLE`` query. -* When there is strong filtration on a small number of columns using ``PREWHERE``. -* In subqueries (since columns that aren't needed for the external query are excluded from subqueries). -In all other cases, we don't recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. - -KILL QUERY -~~~~~~~~~~ - -.. code-block:: sql - - KILL QUERY WHERE [SYNC|ASYNC|TEST] [FORMAT format] - -Tries to finish currently executing queries. -Queries to be finished are selected from ``system.processes`` table according to expression after WHERE term. - -Examples: - -.. code-block:: sql - - KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' - -Finishes all queries with specified query_id. - -.. code-block:: sql - - KILL QUERY WHERE user='username' SYNC - -Synchronously finishes all queries of user ``username``. - -Readonly users can kill only own queries. diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md new file mode 100644 index 00000000000..9049a293c4e --- /dev/null +++ b/docs/en/query_language/syntax.md @@ -0,0 +1,106 @@ +# Syntax + +There are two types of parsers in the system: the full SQL parser (a recursive descent parser), and the data format parser (a fast stream parser). +In all cases except the INSERT query, only the full SQL parser is used. +The INSERT query uses both parsers: + +```sql +INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') +``` + +The `INSERT INTO t VALUES` fragment is parsed by the full parser, and the data `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` is parsed by the fast stream parser. +Data can have any format. When a query is received, the server calculates no more than `max_query_size` bytes of the request in RAM (by default, 1 MB), and the rest is stream parsed. +This means the system doesn't have problems with large INSERT queries, like MySQL does. + +When using the Values format in an INSERT query, it may seem that data is parsed the same as expressions in a SELECT query, but this is not true. The Values format is much more limited. + +Next we will cover the full parser. For more information about format parsers, see the section "Formats". + +## Spaces + +There may be any number of space symbols between syntactical constructions (including the beginning and end of a query). Space symbols include the space, tab, line feed, CR, and form feed. + +## Comments + +SQL-style and C-style comments are supported. +SQL-style comments: from `--` to the end of the line. The space after `--` can be omitted. +Comments in C-style: from `/*` to `*/`. These comments can be multiline. Spaces are not required here, either. + +## Keywords + +Keywords (such as `SELECT`) are not case-sensitive. Everything else (column names, functions, and so on), in contrast to standard SQL, is case-sensitive. Keywords are not reserved (they are just parsed as keywords in the corresponding context). + +## Identifiers + +Identifiers (column names, functions, and data types) can be quoted or non-quoted. +Non-quoted identifiers start with a Latin letter or underscore, and continue with a Latin letter, underscore, or number. In other words, they must match the regex `^[a-zA-Z_][0-9a-zA-Z_]*$`. Examples: `x, _1, X_y__Z123_.` +Квотированные идентификаторы расположены в обратных кавычках `` `id ``\` (the same as in MySQL), and can indicate any set of bytes (non-empty). In addition, symbols (for example, the reverse quotation mark) inside this type of identifier can be backslash-escaped. Escaping rules are the same as for string literals (see below). +We recommend using identifiers that do not need to be quoted. + +## Literals + +There are numeric literals, string literals, and compound literals. + +### Numeric literals + +A numeric literal tries to be parsed: + +- first as a 64-bit signed number, using the 'strtoull' function. +- If unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function. +- If unsuccessful, as a floating-point number using the 'strtod' function. +- Otherwise, an error is returned. + +The corresponding value will have the smallest type that the value fits in. +For example, 1 is parsed as UInt8, but 256 is parsed as UInt16. For more information, see "Data types". + +Examples: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. + +### String literals + +Only string literals in single quotes are supported. The enclosed characters can be backslash-escaped. The following escape sequences have a corresponding special value: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. In all other cases, escape sequences in the format `\c`, where "c" is any character, are converted to "c". This means that you can use the sequences `\'`and`\\`. The value will have the String type. + +The minimum set of characters that you need to escape in string literals: `'` and `\`. + +### Compound literals + +Constructions are supported for arrays: `[1, 2, 3]` and tuples: `(1, 'Hello, world!', 2)`.. +Actually, these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively. +For more information, see the section "Operators2". +An array must consist of at least one item, and a tuple must have at least two items. +Tuples have a special purpose for use in the IN clause of a SELECT query. Tuples can be obtained as the result of a query, but they can't be saved to a database (with the exception of Memory-type tables). + +## Functions + +Functions are written like an identifier with a list of arguments (possibly empty) in brackets. In contrast to standard SQL, the brackets are required, even for an empty arguments list. Example: `now()`. +There are regular and aggregate functions (see the section "Aggregate functions"). Some aggregate functions can contain two lists of arguments in brackets. Example: ` quantile (0.9) (x)`. These aggregate functions are called "parametric" functions, and the arguments in the first list are called "parameters". The syntax of aggregate functions without parameters is the same as for regular functions. + +## Operators + +Operators are converted to their corresponding functions during query parsing, taking their priority and associativity into account. +For example, the expression `1 + 2 * 3 + 4` is transformed to `plus(plus(1, multiply(2, 3)), 4)`. +For more information, see the section "Operators" below. + +## Data types and database table engines + +Data types and table engines in the `CREATE` query are written the same way as identifiers or functions. In other words, they may or may not contain an arguments list in brackets. For more information, see the sections "Data types," "Table engines," and "CREATE". + +## Synonyms + +In the SELECT query, expressions can specify synonyms using the AS keyword. Any expression is placed to the left of AS. The identifier name for the synonym is placed to the right of AS. As opposed to standard SQL, synonyms are not only declared on the top level of expressions: + +```sql +SELECT (1 AS n) + 2, n +``` + +In contrast to standard SQL, synonyms can be used in all parts of a query, not just `SELECT`. + +## Asterisk + +In a `SELECT` query, an asterisk can replace the expression. For more information, see the section "SELECT". + +## Expressions + +An expression is a function, identifier, literal, application of an operator, expression in brackets, subquery, or asterisk. It can also contain a synonym. +A list of expressions is one or more expressions separated by commas. +Functions and operators, in turn, can have expressions as arguments. + diff --git a/docs/en/query_language/syntax.rst b/docs/en/query_language/syntax.rst deleted file mode 100644 index 576e09111d9..00000000000 --- a/docs/en/query_language/syntax.rst +++ /dev/null @@ -1,102 +0,0 @@ -Syntax ------- - -There are two types of parsers in the system: a full SQL parser (a recursive descent parser), and a data format parser (a fast stream parser). In all cases except the INSERT query, only the full SQL parser is used. -The INSERT query uses both parsers: - -.. code-block:: sql - - INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') - -The ``INSERT INTO t VALUES`` fragment is parsed by the full parser, and the data ``(1, 'Hello, world'), (2, 'abc'), (3, 'def')`` is parsed by the fast stream parser. -Data can have any format. When a query is received, the server calculates no more than 'max_query_size' bytes of the request in RAM (by default, 1 MB), and the rest is stream parsed. This means the system doesn't have problems with large INSERT queries, like MySQL does. - -When using the Values format in an ``INSERT`` query, it may seem that data is parsed the same as expressions in a SELECT query, but this is not true. The Values format is much more limited. - -Next we will cover the full parser. For more information about format parsers, see the section "Formats". - -Spaces -~~~~~~ -There may be any number of space symbols between syntactical constructions (including the beginning and end of a query). Space symbols include the space, tab, line break, CR, and form feed. - -Comments -~~~~~~~~ -SQL-style and C-style comments are supported. -SQL-style comments: from ``--`` to the end of the line. The space after ``--`` can be omitted. -C-style comments: from ``/*`` to ``*/``. These comments can be multiline. Spaces are not required here, either. - -Keywords -~~~~~~~~ -Keywords (such as SELECT) are not case-sensitive. Everything else (column names, functions, and so on), in contrast to standard SQL, is case-sensitive. Keywords are not reserved (they are just parsed as keywords in the corresponding context). - -Identifiers -~~~~~~~~~~~ -Identifiers (column names, functions, and data types) can be quoted or non-quoted. -Non-quoted identifiers start with a Latin letter or underscore, and continue with a Latin letter, underscore, or number. In other words, they must match the regex ``^[a-zA-Z_][0-9a-zA-Z_]*$``. Examples: ``x``, ``_1``, ``X_y__Z123_``. -Quoted identifiers are placed in reversed quotation marks ```id``` (the same as in MySQL), and can indicate any set of bytes (non-empty). In addition, symbols (for example, the reverse quotation mark) inside this type of identifier can be backslash-escaped. Escaping rules are the same as for string literals (see below). -We recommend using identifiers that do not need to be quoted. - -Literals -~~~~~~~~ -There are numeric literals, string literals, and compound literals. - -Numeric literals -"""""""""""""""" -A numeric literal tries to be parsed: -- first as a 64-bit signed number, using the 'strtoull' function. -- if unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function. -- if unsuccessful, as a floating-point number using the 'strtod' function. -- otherwise, an error is returned. - -The corresponding value will have the smallest type that the value fits in. -For example, 1 is parsed as UInt8, but 256 is parsed as UInt16. For more information, see "Data types". - -Examples: ``1``, ``18446744073709551615``, ``0xDEADBEEF``, ``01``, ``0.1``, ``1e100``, ``-1e-100``, ``inf``, ``nan``. - -String literals -""""""""""""""" -Only string literals in single quotes are supported. The enclosed characters can be backslash-escaped. The following escape sequences have special meanings: ``\b``, ``\f``, ``\r``, ``\n``, ``\t``, ``\0``, ``\a``, ``\v``, ``\xHH``. In all other cases, escape sequences like ``\c``, where c is any character, are transformed to c. This means that the sequences ``\'`` and ``\\`` can be used. The value will have the String type. - -Minimum set of symbols that must be escaped in string literal is ``'`` and ``\``. - -Compound literals -""""""""""""""""" -Constructions are supported for arrays: ``[1, 2, 3]`` and tuples: ``(1, 'Hello, world!', 2)``. -Actually, these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively. For more information, see the section "Operators". -An array must consist of at least one item, and a tuple must have at least two items. -Tuples have a special purpose for use in the IN clause of a SELECT query. Tuples can be obtained as the result of a query, but they can't be saved to a database (with the exception of Memory-type tables). - -Functions -~~~~~~~~~ -Functions are written like an identifier with a list of arguments (possibly empty) in brackets. In contrast to standard SQL, the brackets are required, even for an empty arguments list. Example: ``now()``. -There are regular and aggregate functions (see the section "Aggregate functions"). Some aggregate functions can contain two lists of arguments in brackets. Example: ``quantile(0.9)(x)``. These aggregate functions are called "parametric" functions, and the arguments in the first list are called "parameters". The syntax of aggregate functions without parameters is the same as for regular functions. - -Operators -~~~~~~~~~ -Operators are converted to their corresponding functions during query parsing, taking their priority and associativity into account. -For example, the expression ``1 + 2 * 3 + 4`` is transformed to ``plus(plus(1, multiply(2, 3)), 4)``. -For more information, see the section "Operators" below. - -Data types and database table engines -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Data types and table engines in the ``CREATE`` query are written the same way as identifiers or functions. In other words, they may or may not contain an arguments list in brackets. For more information, see the sections "Data types," "Table engines," and "CREATE". - -Synonyms -~~~~~~~~ -In the SELECT query, expressions can specify synonyms using the AS keyword. Any expression is placed to the left of AS. The identifier name for the synonym is placed to the right of AS. As opposed to standard SQL, synonyms are not only declared on the top level of expressions: - -.. code-block:: sql - - SELECT (1 AS n) + 2, n - -In contrast to standard SQL, synonyms can be used in all parts of a query, not just ``SELECT``. - -Asterisk -~~~~~~~~ -In a ``SELECT`` query, an asterisk can replace the expression. For more information, see the section "SELECT". - -Expressions -~~~~~~~~~~~ -An expression is a function, identifier, literal, application of an operator, expression in brackets, subquery, or asterisk. It can also contain a synonym. -A list of expressions is one or more expressions separated by commas. -Functions and operators, in turn, can have expressions as arguments. diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md new file mode 100644 index 00000000000..d847a95102b --- /dev/null +++ b/docs/en/roadmap.md @@ -0,0 +1,19 @@ +# Roadmap + +## Q3 2017 + +- ` SYSTEM queries` +- Limit the number of parallel downloads from replicas +- Finish support for `NULL` +- `SELECT db.table.column` + +## Q4 2017 + +- An arbitrary key for partitioning engines of the MergeTree family +- Enhance the ` JOIN syntax` for compatibility with the SQL standard +- Resource pools for queries (CPU, disk I/O, and network bandwidth) + +## Q1 2018 + +- Initial support for `UPDATE` and `DELETE` + diff --git a/docs/en/roadmap.rst b/docs/en/roadmap.rst deleted file mode 100644 index f4247d3b2cf..00000000000 --- a/docs/en/roadmap.rst +++ /dev/null @@ -1,20 +0,0 @@ -Roadmap -======= - -Q3 2017 -------- -* ``SYSTEM`` queries -* Limit on parallel replica downloads -* Finalize ``NULL`` support -* ``SELECT db.table.column`` - -Q4 2017 -------- -* Arbitrary partitioning key for MergeTree engine family -* Better compliance of ``JOIN`` syntax with SQL standard -* Resource pools for queries (CPU, disk I/O, network bandwidth) - -Q1 2018 -------- -* Basic support for ``UPDATE`` and ``DELETE`` - diff --git a/docs/en/system_tables/index.rst b/docs/en/system_tables/index.md similarity index 93% rename from docs/en/system_tables/index.rst rename to docs/en/system_tables/index.md index b0690f4d29d..71c4c058184 100644 --- a/docs/en/system_tables/index.rst +++ b/docs/en/system_tables/index.md @@ -1,5 +1,4 @@ -System tables -============= +# System tables System tables are used for implementing part of the system's functionality, and for providing access to information about how the system is working. You can't delete a system table (but you can perform DETACH). @@ -7,7 +6,10 @@ System tables don't have files with data on the disk or files with metadata. The System tables are read-only. System tables are located in the 'system' database. +```eval_rst .. toctree:: :glob: * +``` + diff --git a/docs/en/system_tables/system.asynchronous_metrics.md b/docs/en/system_tables/system.asynchronous_metrics.md new file mode 100644 index 00000000000..4c2c4877d66 --- /dev/null +++ b/docs/en/system_tables/system.asynchronous_metrics.md @@ -0,0 +1,8 @@ + + +# system.asynchronous_metrics + +Contain metrics used for profiling and monitoring. +They usually reflect the number of events currently in the system, or the total resources consumed by the system. +Example: The number of SELECT queries currently running; the amount of memory in use.`system.asynchronous_metrics`and`system.metrics` differ in their sets of metrics and how they are calculated. + diff --git a/docs/en/system_tables/system.asynchronous_metrics.rst b/docs/en/system_tables/system.asynchronous_metrics.rst deleted file mode 100644 index 3fa9b49403b..00000000000 --- a/docs/en/system_tables/system.asynchronous_metrics.rst +++ /dev/null @@ -1,5 +0,0 @@ -system.asynchronous_metrics ---------------------------- - -Like system.events, but show info about currently executing events or consuming resources. -Example: The number of currently executing SELECT queries; memory consumption of the system. diff --git a/docs/en/system_tables/system.clusters.md b/docs/en/system_tables/system.clusters.md new file mode 100644 index 00000000000..c0bc3dd13fa --- /dev/null +++ b/docs/en/system_tables/system.clusters.md @@ -0,0 +1,16 @@ +# system.clusters + +Contains information about clusters available in the config file and the servers in them. +Columns: + +```text +cluster String – Cluster name. +shard_num UInt32 – Number of a shard in the cluster, starting from 1. +shard_weight UInt32 – Relative weight of a shard when writing data. +replica_num UInt32 – Number of a replica in the shard, starting from 1. +host_name String – Host name as specified in the config. +host_address String – Host's IP address obtained from DNS. +port UInt16 – The port used to access the server. +user String – The username to use for connecting to the server. +``` + diff --git a/docs/en/system_tables/system.clusters.rst b/docs/en/system_tables/system.clusters.rst deleted file mode 100644 index d55d01a02e6..00000000000 --- a/docs/en/system_tables/system.clusters.rst +++ /dev/null @@ -1,16 +0,0 @@ -system.clusters ---------------- - -Contains information about clusters available in the config file and the servers in them. -Columns: - -.. code-block:: text - - cluster String - Cluster name. - shard_num UInt32 - Number of a shard in the cluster, starting from 1. - shard_weight UInt32 - Relative weight of a shard when writing data. - replica_num UInt32 - Number of a replica in the shard, starting from 1. - host_name String - Host name as specified in the config. - host_address String - Host's IP address obtained from DNS. - port UInt16 - The port used to access the server. - user String - The username to use for connecting to the server. diff --git a/docs/en/system_tables/system.columns.md b/docs/en/system_tables/system.columns.md new file mode 100644 index 00000000000..975b84fe9d4 --- /dev/null +++ b/docs/en/system_tables/system.columns.md @@ -0,0 +1,14 @@ +# system.columns + +Contains information about the columns in all tables. +You can use this table to get information similar to `DESCRIBE TABLE`, but for multiple tables at once. + +```text +database String - Name of the database the table is located in. +table String - Table name. +name String - Column name. +type String - Column type. +default_type String - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined. +default_expression String - Expression for the default value, or an empty string if it is not defined. +``` + diff --git a/docs/en/system_tables/system.columns.rst b/docs/en/system_tables/system.columns.rst deleted file mode 100644 index 70f937af07f..00000000000 --- a/docs/en/system_tables/system.columns.rst +++ /dev/null @@ -1,15 +0,0 @@ -system.columns --------------- - -Contains information about the columns in all tables. -You can use this table to get information similar to ``DESCRIBE TABLE``, but for multiple tables at once. - -.. code-block:: text - - database String - Name of the database the table is located in. - table String - Table name. - name String - Column name. - type String - Column type. - default_type String - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined. - default_expression String - Expression for the default value, or an empty string if it is not defined. - diff --git a/docs/en/system_tables/system.databases.md b/docs/en/system_tables/system.databases.md new file mode 100644 index 00000000000..ea062907c11 --- /dev/null +++ b/docs/en/system_tables/system.databases.md @@ -0,0 +1,6 @@ +# system.databases + +This table contains a single String column called 'name' – the name of a database. +Each database that the server knows about has a corresponding entry in the table. +This system table is used for implementing the `SHOW DATABASES` query. + diff --git a/docs/en/system_tables/system.databases.rst b/docs/en/system_tables/system.databases.rst deleted file mode 100644 index 62e55862cb8..00000000000 --- a/docs/en/system_tables/system.databases.rst +++ /dev/null @@ -1,6 +0,0 @@ -system.databases ----------------- - -This table contains a single String column called 'name' - the name of a database. -Each database that the server knows about has a corresponding entry in the table. -This system table is used for implementing the ``SHOW DATABASES`` query. diff --git a/docs/en/system_tables/system.dictionaries.md b/docs/en/system_tables/system.dictionaries.md new file mode 100644 index 00000000000..f3c9929d38e --- /dev/null +++ b/docs/en/system_tables/system.dictionaries.md @@ -0,0 +1,24 @@ +# system.dictionaries + +Contains information about external dictionaries. + +Columns: + +```text +name String – Dictionary name. +type String – Dictionary type: Flat, Hashed, Cache. +origin String – Path to the config file where the dictionary is described.attribute. +names Array(String) – Array of attribute names provided by the dictionary. +attribute.types Array(String) – Corresponding array of attribute types provided by the dictionary. +has_hierarchy UInt8 – Whether the dictionary is hierarchical. +bytes_allocated UInt64 – The amount of RAM used by the dictionary. +hit_rate Float64 – For cache dictionaries, the percent of usage for which the value was in the cache. +element_count UInt64 – The number of items stored in the dictionary. +load_factor Float64 – The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). +creation_time DateTime – Time spent for the creation or last successful reload of the dictionary. +last_exception String – Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. +source String – Text describing the data source for the dictionary. +``` + +Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is. + diff --git a/docs/en/system_tables/system.dictionaries.rst b/docs/en/system_tables/system.dictionaries.rst deleted file mode 100644 index dbf2781fd5f..00000000000 --- a/docs/en/system_tables/system.dictionaries.rst +++ /dev/null @@ -1,24 +0,0 @@ -system.dictionaries -------------------- - -Contains information about external dictionaries. - -Columns: - -.. code-block:: text - - name String - Dictionary name. - type String - Dictionary type: Flat, Hashed, Cache. - origin String - Path to the config file where the dictionary is described. - attribute.names Array(String) - Array of attribute names provided by the dictionary. - attribute.types Array(String) - Corresponding array of attribute types provided by the dictionary. - has_hierarchy UInt8 - Whether the dictionary is hierarchical. - bytes_allocated UInt64 - The amount of RAM used by the dictionary. - hit_rate Float64 - For cache dictionaries, the percent of usage for which the value was in the cache. - element_count UInt64 - The number of items stored in the dictionary. - load_factor Float64 - The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). - creation_time DateTime - Time spent for the creation or last successful reload of the dictionary. - last_exception String - Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. - source String - Text describing the data source for the dictionary. - -Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is. diff --git a/docs/en/system_tables/system.events.rst b/docs/en/system_tables/system.events.md similarity index 56% rename from docs/en/system_tables/system.events.rst rename to docs/en/system_tables/system.events.md index feaf96a1f44..87c0700e6d7 100644 --- a/docs/en/system_tables/system.events.rst +++ b/docs/en/system_tables/system.events.md @@ -1,6 +1,8 @@ -system.events -------------- + + +# system.events Contains information about the number of events that have occurred in the system. This is used for profiling and monitoring purposes. Example: The number of processed SELECT queries. -Columns: 'event String' - the event name, and 'value UInt64' - the quantity. +Columns: 'event String' – the event name, and 'value UInt64' – the quantity. + diff --git a/docs/en/system_tables/system.functions.md b/docs/en/system_tables/system.functions.md new file mode 100644 index 00000000000..ac550acc14b --- /dev/null +++ b/docs/en/system_tables/system.functions.md @@ -0,0 +1,11 @@ +# system.functions + +Contains information about normal and aggregate functions. + +Columns: + +```text +name String – Function name. +is_aggregate UInt8 – Whether it is an aggregate function. +``` + diff --git a/docs/en/system_tables/system.functions.rst b/docs/en/system_tables/system.functions.rst deleted file mode 100644 index d864f6d990b..00000000000 --- a/docs/en/system_tables/system.functions.rst +++ /dev/null @@ -1,10 +0,0 @@ -system.functions ----------------- - -Contains information about normal and aggregate functions. -Columns: - -.. code-block:: text - - name String - Function name. - is_aggregate UInt8 - Whether it is an aggregate function. diff --git a/docs/en/system_tables/system.merges.md b/docs/en/system_tables/system.merges.md new file mode 100644 index 00000000000..0a10e4a5a8c --- /dev/null +++ b/docs/en/system_tables/system.merges.md @@ -0,0 +1,21 @@ +# system.merges + +Contains information about merges currently in process for tables in the MergeTree family. + +Columns: + +```text +database String - Name of the database the table is located in. +table String - Name of the table. +elapsed Float64 - Time in seconds since the merge started. +progress Float64 - Percent of progress made, from 0 to 1. +num_parts UInt64 - Number of parts to merge. +result_part_name String - Name of the part that will be formed as the result of the merge. +total_size_bytes_compressed UInt64 - Total size of compressed data in the parts being merged. +total_size_marks UInt64 - Total number of marks in the parts being merged. +bytes_read_uncompressed UInt64 - Amount of bytes read, decompressed. +rows_read UInt64 - Number of rows read. +bytes_written_uncompressed UInt64 - Amount of bytes written, uncompressed. +rows_written UInt64 - Number of rows written. +``` + diff --git a/docs/en/system_tables/system.merges.rst b/docs/en/system_tables/system.merges.rst deleted file mode 100644 index 4286e199781..00000000000 --- a/docs/en/system_tables/system.merges.rst +++ /dev/null @@ -1,20 +0,0 @@ -system.merges -------------- -Contains information about merges currently in process for tables in the MergeTree family. - -Columns: - -.. code-block:: text - - database String - Name of the database the table is located in. - table String - Name of the table. - elapsed Float64 - Time in seconds since the merge started. - progress Float64 - Percent of progress made, from 0 to 1. - num_parts UInt64 - Number of parts to merge. - result_part_name String - Name of the part that will be formed as the result of the merge. - total_size_bytes_compressed UInt64 - Total size of compressed data in the parts being merged. - total_size_marks UInt64 - Total number of marks in the parts being merged. - bytes_read_uncompressed UInt64 - Amount of bytes read, decompressed. - rows_read UInt64 - Number of rows read. - bytes_written_uncompressed UInt64 - Amount of bytes written, uncompressed. - rows_written UInt64 - Number of rows written. diff --git a/docs/en/system_tables/system.metrics.md b/docs/en/system_tables/system.metrics.md new file mode 100644 index 00000000000..39acbaae0dc --- /dev/null +++ b/docs/en/system_tables/system.metrics.md @@ -0,0 +1,4 @@ + + +# system.metrics + diff --git a/docs/en/system_tables/system.metrics.rst b/docs/en/system_tables/system.metrics.rst deleted file mode 100644 index dee53b399e6..00000000000 --- a/docs/en/system_tables/system.metrics.rst +++ /dev/null @@ -1,2 +0,0 @@ -system.metrics --------------- diff --git a/docs/en/system_tables/system.numbers.rst b/docs/en/system_tables/system.numbers.md similarity index 89% rename from docs/en/system_tables/system.numbers.rst rename to docs/en/system_tables/system.numbers.md index baaef83daab..42bbe391e2d 100644 --- a/docs/en/system_tables/system.numbers.rst +++ b/docs/en/system_tables/system.numbers.md @@ -1,6 +1,6 @@ -system.numbers --------------- +# system.numbers This table contains a single UInt64 column named 'number' that contains almost all the natural numbers starting from zero. You can use this table for tests, or if you need to do a brute force search. Reads from this table are not parallelized. + diff --git a/docs/en/system_tables/system.numbers_mt.rst b/docs/en/system_tables/system.numbers_mt.md similarity index 76% rename from docs/en/system_tables/system.numbers_mt.rst rename to docs/en/system_tables/system.numbers_mt.md index b5569748b2a..9b8064182f7 100644 --- a/docs/en/system_tables/system.numbers_mt.rst +++ b/docs/en/system_tables/system.numbers_mt.md @@ -1,5 +1,5 @@ -system.numbers_mt ------------------ +# system.numbers_mt The same as 'system.numbers' but reads are parallelized. The numbers can be returned in any order. Used for tests. + diff --git a/docs/en/system_tables/system.one.rst b/docs/en/system_tables/system.one.md similarity index 90% rename from docs/en/system_tables/system.one.rst rename to docs/en/system_tables/system.one.md index 8a4c1821fb8..cb170010104 100644 --- a/docs/en/system_tables/system.one.rst +++ b/docs/en/system_tables/system.one.md @@ -1,6 +1,6 @@ -system.one ----------- +# system.one This table contains a single row with a single 'dummy' UInt8 column containing the value 0. This table is used if a SELECT query doesn't specify the FROM clause. This is similar to the DUAL table found in other DBMSs. + diff --git a/docs/en/system_tables/system.parts.md b/docs/en/system_tables/system.parts.md new file mode 100644 index 00000000000..402908e42f2 --- /dev/null +++ b/docs/en/system_tables/system.parts.md @@ -0,0 +1,29 @@ +# system.parts + +Contains information about parts of a table in the [MergeTree](../table_engines/mergetree.md#table_engines-mergetree) family. + +Each row describes one part of the data. + +Columns: + +- partition (String) – The partition name. YYYYMM format. To learn what a partition is, see the description of the [ALTER](../query_language/queries.md#query_language_queries_alter) query. +- name (String) – Name of the data part. +- active (UInt8) – Indicates whether the part is active. If a part is active, it is used in a table; otherwise, it will be deleted. Inactive data parts remain after merging. +- marks (UInt64) – The number of marks. To get the approximate number of rows in a data part, multiply ``marks`` by the index granularity (usually 8192). +- marks_size (UInt64) – The size of the file with marks. +- rows (UInt64) – The number of rows. +- bytes (UInt64) – The number of bytes when compressed. +- modification_time (DateTime) – The modification time of the directory with the data part. This usually corresponds to the time of data part creation.| +- remove_time (DateTime) – The time when the data part became inactive. +- refcount (UInt32) – The number of places where the data part is used. A value greater than 2 indicates that the data part is used in queries or merges. +- min_date (Date) – The minimum value of the date key in the data part. +- max_date (Date) – The maximum value of the date key in the data part. +- min_block_number (UInt64) – The minimum number of data parts that make up the current part after merging. +- max_block_number (UInt64) – The maximum number of data parts that make up the current part after merging. +- level (UInt32) – Depth of the merge tree. If a merge was not performed, ``level=0``. +- primary_key_bytes_in_memory (UInt64) – The amount of memory (in bytes) used by primary key values. +- primary_key_bytes_in_memory_allocated (UInt64) – The amount of memory (in bytes) reserved for primary key values. +- database (String) – Name of the database. +- table (String) – Name of the table. +- engine (String) – Name of the table engine without parameters. + diff --git a/docs/en/system_tables/system.parts.rst b/docs/en/system_tables/system.parts.rst deleted file mode 100644 index a04e133b3d9..00000000000 --- a/docs/en/system_tables/system.parts.rst +++ /dev/null @@ -1,20 +0,0 @@ -system.parts ------------- -Contains information about parts of a table in the MergeTree family. - -Columns: - -.. code-block:: text - - database String - Name of the database where the table that this part belongs to is located. - table String - Name of the table that this part belongs to. - engine String - Name of the table engine, without parameters. - partition String - Name of the partition, in the format YYYYMM. - name String - Name of the part. - replicated UInt8 - Whether the part belongs to replicated data. - active UInt8 - Whether the part is used in a table, or is no longer needed and will be deleted soon. Inactive parts remain after merging. - marks UInt64 - Number of marks - multiply by the index granularity (usually 8192) to get the approximate number of rows in the part. - bytes UInt64 - Number of bytes when compressed. - modification_time DateTime - Time the directory with the part was modified. Usually corresponds to the part's creation time. - remove_time DateTime - For inactive parts only - the time when the part became inactive. - refcount UInt32 - The number of places where the part is used. A value greater than 2 indicates that this part participates in queries or merges. diff --git a/docs/en/system_tables/system.processes.md b/docs/en/system_tables/system.processes.md new file mode 100644 index 00000000000..b9ad6a44e81 --- /dev/null +++ b/docs/en/system_tables/system.processes.md @@ -0,0 +1,25 @@ +# system.processes + +This system table is used for implementing the `SHOW PROCESSLIST` query. +Columns: + +```text +user String – Name of the user who made the request. For distributed query processing, this is the user who helped the requestor server send the query to this server, not the user who made the distributed request on the requestor server. + +address String – The IP address that the query was made from. The same is true for distributed query processing. + +elapsed Float64 – The time in seconds since request execution started. + +rows_read UInt64 – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. + +bytes_read UInt64 – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. + +total_rows_approx UInt64 – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. + +memory_usage UInt64 – Memory consumption by the query. It might not include some types of dedicated memory. + +query String – The query text. For INSERT, it doesn't include the data to insert. + +query_id String - The query ID, if defined. +``` + diff --git a/docs/en/system_tables/system.processes.rst b/docs/en/system_tables/system.processes.rst deleted file mode 100644 index cb401224c54..00000000000 --- a/docs/en/system_tables/system.processes.rst +++ /dev/null @@ -1,25 +0,0 @@ -system.processes ----------------- - -This system table is used for implementing the ``SHOW PROCESSLIST`` query. -Columns: - -.. code-block:: text - - user String - Name of the user who made the request. For distributed query processing, this is the user who helped the requestor server send the query to this server, not the user who made the distributed request on the requestor server. - - address String - The IP address the request was made from. The same for distributed processing. - - elapsed Float64 - The time in seconds since request execution started. - - rows_read UInt64 - The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - - bytes_read UInt64 - The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - - total_rows_approx UInt64 - The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. - - memory_usage UInt64 - How much memory the request uses. It might not include some types of dedicated memory. - - query String - The query text. For INSERT, it doesn't include the data to insert. - - query_id String - Query ID, if defined. diff --git a/docs/en/system_tables/system.replicas.md b/docs/en/system_tables/system.replicas.md new file mode 100644 index 00000000000..ec1341198dc --- /dev/null +++ b/docs/en/system_tables/system.replicas.md @@ -0,0 +1,117 @@ +# system.replicas + +Contains information and status for replicated tables residing on the local server. +This table can be used for monitoring. The table contains a row for every Replicated\* table. + +Example: + +```sql +SELECT * +FROM system.replicas +WHERE table = 'visits' +FORMAT Vertical +``` + +```text +Row 1: +────── +database: merge +table: visits +engine: ReplicatedCollapsingMergeTree +is_leader: 1 +is_readonly: 0 +is_session_expired: 0 +future_parts: 1 +parts_to_check: 0 +zookeeper_path: /clickhouse/tables/01-06/visits +replica_name: example01-06-1.yandex.ru +replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru +columns_version: 9 +queue_size: 1 +inserts_in_queue: 0 +merges_in_queue: 1 +log_max_index: 596273 +log_pointer: 596274 +total_replicas: 2 +active_replicas: 2 +``` + +Columns: + +```text +database: database name +table: table name +engine: table engine name +is_leader: whether the replica is the leader + +Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform. +Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. + +is_readonly: Whether the replica is in read-only mode.This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. + +is_session_expired: Whether the ZK session expired. +Basically, the same thing as is_readonly. + +future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. + +parts_to_check: The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged. + +zookeeper_path: The path to the table data in ZK. +replica_name: Name of the replica in ZK. Different replicas of the same table have different names. +replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path. + +columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. + +queue_size: Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions. +Normally coincides with future_parts. + +inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong. + +merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time. + +The next 4 columns have a non-null value only if the ZK session is active. + +log_max_index: Maximum entry number in the log of general activity. +log_pointer: Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. If log_pointer is much smaller than log_max_index, something is wrong. + +total_replicas: Total number of known replicas of this table. +active_replicas: Number of replicas of this table that have a ZK session (the number of active replicas). +``` + +If you request all the columns, the table may work a bit slowly, since several reads from ZK are made for each row. +If you don't request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. + +For example, you can check that everything is working correctly like this: + +```sql +SELECT + database, + table, + is_leader, + is_readonly, + is_session_expired, + future_parts, + parts_to_check, + columns_version, + queue_size, + inserts_in_queue, + merges_in_queue, + log_max_index, + log_pointer, + total_replicas, + active_replicas +FROM system.replicas +WHERE + is_readonly + OR is_session_expired + OR future_parts > 20 + OR parts_to_check > 10 + OR queue_size > 20 + OR inserts_in_queue > 10 + OR log_max_index - log_pointer > 10 + OR total_replicas < 2 + OR active_replicas < total_replicas +``` + +If this query doesn't return anything, it means that everything is fine. + diff --git a/docs/en/system_tables/system.replicas.rst b/docs/en/system_tables/system.replicas.rst deleted file mode 100644 index ee4d2ea3c14..00000000000 --- a/docs/en/system_tables/system.replicas.rst +++ /dev/null @@ -1,116 +0,0 @@ -system.replicas ---------------- - -Contains information and status for replicated tables residing on the local server. This table can be used for monitoring. The table contains a row for every Replicated* table. - -Example: - -.. code-block:: text - - SELECT * - FROM system.replicas - WHERE table = 'visits' - FORMAT Vertical - - Row 1: - ────── - database: merge - table: visits - engine: ReplicatedCollapsingMergeTree - is_leader: 1 - is_readonly: 0 - is_session_expired: 0 - future_parts: 1 - parts_to_check: 0 - zookeeper_path: /clickhouse/tables/01-06/visits - replica_name: example01-06-1.yandex.ru - replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru - columns_version: 9 - queue_size: 1 - inserts_in_queue: 0 - merges_in_queue: 1 - log_max_index: 596273 - log_pointer: 596274 - total_replicas: 2 - active_replicas: 2 - -Columns: - -.. code-block:: text - - database: Database name. - table: Table name. - engine: Table engine name. - - is_leader: Whether the replica is the leader. - Only one replica can be the leader at a time. The leader is responsible for selecting background merges to perform. - Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. - - is_readonly: Whether the replica is in read-only mode. - This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. - - is_session_expired: Whether the session with ZK has expired. - Basically the same as 'is_readonly'. - - future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. - - parts_to_check: The number of data parts in the queue for verification. - A part is put in the verification queue if there is suspicion that it might be damaged. - - zookeeper_path: Path to table data in ZK. - replica_name: Replica name in ZK. Different replicas of the same table have different names. - replica_path: Path to replica data in ZK. The same as concatenating 'zookeeper_path/replicas/replica_path'. - - columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. - - queue_size: Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions. It usually coincides with 'future_parts'. - - inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If this number is large, it means something is wrong. - - merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than one for a long time. - - The next 4 columns have a non-zero value only where there is an active session with ZK. - - log_max_index: Maximum entry number in the log of general activity. - log_pointer: Maximum entry number from the log of general activity that the replica copied to its queue for execution, plus one. - If log_pointer is much smaller than log_max_index, something is wrong. - - total_replicas: The total number of known replicas of this table. - active_replicas: The number of replicas of this table that have a session in ZK (i.e., the number of functioning replicas).к - -If you request all the columns, the table may work a bit slowly, since several reads from ZK are made for each row. -If you don't request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. - -For example, you can check that everything is working correctly like this: - -.. code-block:: sql - - SELECT - database, - table, - is_leader, - is_readonly, - is_session_expired, - future_parts, - parts_to_check, - columns_version, - queue_size, - inserts_in_queue, - merges_in_queue, - log_max_index, - log_pointer, - total_replicas, - active_replicas - FROM system.replicas - WHERE - is_readonly - OR is_session_expired - OR future_parts > 20 - OR parts_to_check > 10 - OR queue_size > 20 - OR inserts_in_queue > 10 - OR log_max_index - log_pointer > 10 - OR total_replicas < 2 - OR active_replicas < total_replicas - -If this query doesn't return anything, it means that everything is fine. diff --git a/docs/en/system_tables/system.settings.md b/docs/en/system_tables/system.settings.md new file mode 100644 index 00000000000..a055135ebcf --- /dev/null +++ b/docs/en/system_tables/system.settings.md @@ -0,0 +1,30 @@ +# system.settings + +Contains information about settings that are currently in use. +I.e. used for executing the query you are using to read from the system.settings table). + +Columns: + +```text +name String – Setting name. +value String – Setting value. +changed UInt8 -–Whether the setting was explicitly defined in the config or explicitly changed. +``` + +Example: + +```sql +SELECT * +FROM system.settings +WHERE changed +``` + +```text +┌─name───────────────────┬─value───────┬─changed─┐ +│ max_threads │ 8 │ 1 │ +│ use_uncompressed_cache │ 0 │ 1 │ +│ load_balancing │ random │ 1 │ +│ max_memory_usage │ 10000000000 │ 1 │ +└────────────────────────┴─────────────┴─────────┘ +``` + diff --git a/docs/en/system_tables/system.settings.rst b/docs/en/system_tables/system.settings.rst deleted file mode 100644 index e15a2a1f295..00000000000 --- a/docs/en/system_tables/system.settings.rst +++ /dev/null @@ -1,29 +0,0 @@ -system.settings ---------------- - -Contains information about settings that are currently in use (i.e. used for executing the query you are using to read from the system.settings table). - -Columns: - -.. code-block:: text - - name String - Setting name. - value String - Setting value. - changed UInt8 - Whether the setting was explicitly defined in the config or explicitly changed. - -Example: - -.. code-block:: sql - - SELECT * - FROM system.settings - WHERE changed - -.. code-block:: text - - ┌─name───────────────────┬─value───────┬─changed─┐ - │ max_threads │ 8 │ 1 │ - │ use_uncompressed_cache │ 0 │ 1 │ - │ load_balancing │ random │ 1 │ - │ max_memory_usage │ 10000000000 │ 1 │ - └────────────────────────┴─────────────┴─────────┘ diff --git a/docs/en/system_tables/system.tables.rst b/docs/en/system_tables/system.tables.md similarity index 76% rename from docs/en/system_tables/system.tables.rst rename to docs/en/system_tables/system.tables.md index 0e7536f9ec9..3d0af2ed603 100644 --- a/docs/en/system_tables/system.tables.rst +++ b/docs/en/system_tables/system.tables.md @@ -1,7 +1,7 @@ -system.tables -------------- +# system.tables -This table contains the String columns 'database', 'name', and 'engine' and DateTime column metadata_modification_time. +This table contains the String columns 'database', 'name', and 'engine' and the DateTime column 'metadata_modification_time'. Each table that the server knows about is entered in the 'system.tables' table. There is an issue: table engines are specified without parameters. This system table is used for implementing SHOW TABLES queries. + diff --git a/docs/en/system_tables/system.zookeeper.md b/docs/en/system_tables/system.zookeeper.md new file mode 100644 index 00000000000..c456e7d9207 --- /dev/null +++ b/docs/en/system_tables/system.zookeeper.md @@ -0,0 +1,73 @@ +# system.zookeeper + +Allows reading data from the ZooKeeper cluster defined in the config. +The query must have a 'path' equality condition in the WHERE clause. This is the path in ZooKeeper for the children that you want to get data for. + +The query `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` outputs data for all children on the `/clickhouse` node. +To output data for all root nodes, write path = '/'. +If the path specified in 'path' doesn't exist, an exception will be thrown. + +Columns: + +```text +name String - Name of the node. +path String - Path to the node. +value String - Value of the node. +dataLength Int32 - Size of the value. +numChildren Int32 - Number of children. +czxid Int64 - ID of the transaction that created the node. +mzxid Int64 - ID of the transaction that last changed the node. +pzxid Int64 - ID of the transaction that last added or removed children. +ctime DateTime - Time of node creation. +mtime DateTime - Time of the last node modification. +version Int32 - Node version - the number of times the node was changed. +cversion Int32 - Number of added or removed children. +aversion Int32 - Number of changes to ACL. +ephemeralOwner Int64 - For ephemeral nodes, the ID of the session that owns this node. +``` + +Example: + +```sql +SELECT * +FROM system.zookeeper +WHERE path = '/clickhouse/tables/01-08/visits/replicas' +FORMAT Vertical +``` + +```text +Row 1: +────── +name: example01-08-1.yandex.ru +value: +czxid: 932998691229 +mzxid: 932998691229 +ctime: 2015-03-27 16:49:51 +mtime: 2015-03-27 16:49:51 +version: 0 +cversion: 47 +aversion: 0 +ephemeralOwner: 0 +dataLength: 0 +numChildren: 7 +pzxid: 987021031383 +path: /clickhouse/tables/01-08/visits/replicas + +Row 2: +────── +name: example01-08-2.yandex.ru +value: +czxid: 933002738135 +mzxid: 933002738135 +ctime: 2015-03-27 16:57:01 +mtime: 2015-03-27 16:57:01 +version: 0 +cversion: 37 +aversion: 0 +ephemeralOwner: 0 +dataLength: 0 +numChildren: 7 +pzxid: 987021252247 +path: /clickhouse/tables/01-08/visits/replicas +``` + diff --git a/docs/en/system_tables/system.zookeeper.rst b/docs/en/system_tables/system.zookeeper.rst deleted file mode 100644 index 70bbd74d0f9..00000000000 --- a/docs/en/system_tables/system.zookeeper.rst +++ /dev/null @@ -1,71 +0,0 @@ -system.zookeeper ----------------- - -Allows reading data from the ZooKeeper cluster defined in the config. -The query must have a 'path' equality condition in the WHERE clause. This is the path in ZooKeeper for the children that you want to get data for. - -Query SELECT * FROM system.zookeeper WHERE path = '/clickhouse' outputs data for all children on the /clickhouse node. -To output data for all root nodes, write path = '/'. -If the path specified in 'path' doesn't exist, an exception will be thrown. - -Columns: - -.. code-block:: text - - name String - Name of the node. - path String - Path to the node. - value String - Value of the node. - dataLength Int32 - Size of the value. - numChildren Int32 - Number of children. - czxid Int64 - ID of the transaction that created the node. - mzxid Int64 - ID of the transaction that last changed the node. - pzxid Int64 - ID of the transaction that last added or removed children. - ctime DateTime - Time of node creation. - mtime DateTime - Time of the last node modification. - version Int32 - Node version - the number of times the node was changed. - cversion Int32 - Number of added or removed children. - aversion Int32 - Number of changes to ACL. - ephemeralOwner Int64 - For ephemeral nodes, the ID of the session that owns this node. - -Example: - -.. code-block:: text - - SELECT * - FROM system.zookeeper - WHERE path = '/clickhouse/tables/01-08/visits/replicas' - FORMAT Vertical - - Row 1: - ────── - name: example01-08-1.yandex.ru - value: - czxid: 932998691229 - mzxid: 932998691229 - ctime: 2015-03-27 16:49:51 - mtime: 2015-03-27 16:49:51 - version: 0 - cversion: 47 - aversion: 0 - ephemeralOwner: 0 - dataLength: 0 - numChildren: 7 - pzxid: 987021031383 - path: /clickhouse/tables/01-08/visits/replicas - - Row 2: - ────── - name: example01-08-2.yandex.ru - value: - czxid: 933002738135 - mzxid: 933002738135 - ctime: 2015-03-27 16:57:01 - mtime: 2015-03-27 16:57:01 - version: 0 - cversion: 37 - aversion: 0 - ephemeralOwner: 0 - dataLength: 0 - numChildren: 7 - pzxid: 987021252247 - path: /clickhouse/tables/01-08/visits/replicas diff --git a/docs/en/table_engines/aggregatingmergetree.md b/docs/en/table_engines/aggregatingmergetree.md new file mode 100644 index 00000000000..f4c0acb216c --- /dev/null +++ b/docs/en/table_engines/aggregatingmergetree.md @@ -0,0 +1,81 @@ +# AggregatingMergeTree + +This engine differs from MergeTree in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. + +In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely. + +There is an AggregateFunction data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. +Examples: + +```sql +CREATE TABLE t +( + column1 AggregateFunction(uniq, UInt64), + column2 AggregateFunction(anyIf, String, UInt8), + column3 AggregateFunction(quantiles(0.5, 0.9), UInt64) +) ENGINE = ... +``` + +This type of column stores the state of an aggregate function. + +To get this type of value, use aggregate functions with the `State` suffix. +Example: `uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` – in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value. + +An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump. + +The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. +Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction type`. + +In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result. +As an example, these two queries return the same result: + +```sql +SELECT uniq(UserID) FROM table + +SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP BY RegionID) +``` + +There is an ` AggregatingMergeTree` engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value. + +You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data. + +With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. + +You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views. + +Пример:Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table: + +```sql +CREATE MATERIALIZED VIEW test.basic +ENGINE = AggregatingMergeTree(StartDate, (CounterID, StartDate), 8192) +AS SELECT + CounterID, + StartDate, + sumState(Sign) AS Visits, + uniqState(UserID) AS Users +FROM test.visits +GROUP BY CounterID, StartDate; +``` + +Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated: + +```sql +INSERT INTO test.visits ... +``` + +Performing SELECT from the view using GROUP BY to finish data aggregation: + +```sql +SELECT + StartDate, + sumMerge(Visits) AS Visits, + uniqMerge(Users) AS Users +FROM test.basic +GROUP BY StartDate +ORDER BY StartDate; +``` + +You can create a materialized view like this and assign a normal view to it that finishes data aggregation. + +Note that in most cases, using `AggregatingMergeTree` is not justified, since queries can be run efficiently enough on non-aggregated data. + diff --git a/docs/en/table_engines/aggregatingmergetree.rst b/docs/en/table_engines/aggregatingmergetree.rst deleted file mode 100644 index ec4e6d8bd8a..00000000000 --- a/docs/en/table_engines/aggregatingmergetree.rst +++ /dev/null @@ -1,82 +0,0 @@ -AggregatingMergeTree --------------------- - -This engine differs from ``MergeTree`` in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. - -In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely. - -There is an AggregateFunction data type, which is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. -Examples: - -.. code-block:: sql - - CREATE TABLE t - ( - column1 AggregateFunction(uniq, UInt64), - column2 AggregateFunction(anyIf, String, UInt8), - column3 AggregateFunction(quantiles(0.5, 0.9), UInt64) - ) ENGINE = ... - -This type of column stores the state of an aggregate function. - -To get this type of value, use aggregate functions with the 'State' suffix. -Example: uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming) - in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value. - -An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump. - -The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. -Example: uniqMerge(UserIDState), where UserIDState has the AggregateFunction type. - -In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result. -As an example, these two queries return the same result: - -.. code-block:: sql - - SELECT uniq(UserID) FROM table - - SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP BY RegionID) - -There is an AggregatingMergeTree engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value. - -You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data. - -With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. - -You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views. - -Example: -Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table: - -.. code-block:: sql - - CREATE MATERIALIZED VIEW test.basic - ENGINE = AggregatingMergeTree(StartDate, (CounterID, StartDate), 8192) - AS SELECT - CounterID, - StartDate, - sumState(Sign) AS Visits, - uniqState(UserID) AS Users - FROM test.visits - GROUP BY CounterID, StartDate; - -Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated: - -.. code-block:: sql - - INSERT INTO test.visits ... - -Performing SELECT from the view using GROUP BY to finish data aggregation: - -.. code-block:: sql - - SELECT - StartDate, - sumMerge(Visits) AS Visits, - uniqMerge(Users) AS Users - FROM test.basic - GROUP BY StartDate - ORDER BY StartDate; - -You can create a materialized view like this and assign a normal view to it that finishes data aggregation. - -Note that in most cases, using AggregatingMergeTree is not justified, since queries can be run efficiently enough on non-aggregated data. diff --git a/docs/en/table_engines/buffer.rst b/docs/en/table_engines/buffer.md similarity index 80% rename from docs/en/table_engines/buffer.rst rename to docs/en/table_engines/buffer.md index 353199f6bfb..2098c5f19da 100644 --- a/docs/en/table_engines/buffer.rst +++ b/docs/en/table_engines/buffer.md @@ -1,21 +1,14 @@ -Buffer ------- +# Buffer Buffers the data to write in RAM, periodically flushing it to another table. During the read operation, data is read from the buffer and the other table simultaneously. -.. code-block:: text +```text +Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) +``` - Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) +Engine parameters:database, table – The table to flush data to. Instead of the database name, you can use a constant expression that returns a string.num_layers – Parallelism layer. Physically, the table will be represented as 'num_layers' of independent buffers. Recommended value: 16.min_time, max_time, min_rows, max_rows, min_bytes, and max_bytes are conditions for flushing data from the buffer. -Engine parameters: -database, table - The table to flush data to. Instead of the database name, you can use a constant expression that returns a string. -num_layers - The level of parallelism. Physically, the table will be represented as 'num_layers' of independent buffers. The recommended value is 16. -min_time, max_time, min_rows, max_rows, min_bytes, and max_bytes are conditions for flushing data from the buffer. - -Data is flushed from the buffer and written to the destination table if all the 'min' conditions or at least one 'max' condition are met. -min_time, max_time - Condition for the time in seconds from the moment of the first write to the buffer. -min_rows, max_rows - Condition for the number of rows in the buffer. -min_bytes, max_bytes - Condition for the number of bytes in the buffer. +Data is flushed from the buffer and written to the destination table if all the 'min' conditions or at least one 'max' condition are met.min_time, max_time – Condition for the time in seconds from the moment of the first write to the buffer.min_rows, max_rows – Condition for the number of rows in the buffer.min_bytes, max_bytes – Condition for the number of bytes in the buffer. During the write operation, data is inserted to a 'num_layers' number of random buffers. Or, if the data part to insert is large enough (greater than 'max_rows' or 'max_bytes'), it is written directly to the destination table, omitting the buffer. @@ -23,9 +16,9 @@ The conditions for flushing the data are calculated separately for each of the ' Example: -.. code-block:: sql - - CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) +```sql +CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000) +``` Creating a 'merge.hits_buffer' table with the same structure as 'merge.hits' and using the Buffer engine. When writing to this table, data is buffered in RAM and later written to the 'merge.hits' table. 16 buffers are created. The data in each of them is flushed if either 100 seconds have passed, or one million rows have been written, or 100 MB of data have been written; or if simultaneously 10 seconds have passed and 10,000 rows and 10 MB of data have been written. For example, if just one row has been written, after 100 seconds it will be flushed, no matter what. But if many rows have been written, the data will be flushed sooner. @@ -34,7 +27,7 @@ When the server is stopped, with DROP TABLE or DETACH TABLE, buffer data is also You can set empty strings in single quotation marks for the database and table name. This indicates the absence of a destination table. In this case, when the data flush conditions are reached, the buffer is simply cleared. This may be useful for keeping a window of data in memory. When reading from a Buffer table, data is processed both from the buffer and from the destination table (if there is one). -Note that the Buffer tables does not support an index. In other words, data in the buffer is fully scanned, which might be slow for large buffers. (For data in a subordinate table, the index it supports will be used.) +Note that the Buffer tables does not support an index. In other words, data in the buffer is fully scanned, which might be slow for large buffers. (For data in a subordinate table, the index that it supports will be used.) If the set of columns in the Buffer table doesn't match the set of columns in a subordinate table, a subset of columns that exist in both tables is inserted. @@ -58,3 +51,4 @@ Due to these disadvantages, we can only recommend using a Buffer table in rare c A Buffer table is used when too many INSERTs are received from a large number of servers over a unit of time and data can't be buffered before insertion, which means the INSERTs can't run fast enough. Note that it doesn't make sense to insert data one row at a time, even for Buffer tables. This will only produce a speed of a few thousand rows per second, while inserting larger blocks of data can produce over a million rows per second (see the section "Performance"). + diff --git a/docs/en/table_engines/collapsingmergetree.rst b/docs/en/table_engines/collapsingmergetree.md similarity index 73% rename from docs/en/table_engines/collapsingmergetree.rst rename to docs/en/table_engines/collapsingmergetree.md index e2bacf2512e..dc0928744f6 100644 --- a/docs/en/table_engines/collapsingmergetree.rst +++ b/docs/en/table_engines/collapsingmergetree.md @@ -1,7 +1,8 @@ -CollapsingMergeTree -------------------- +# CollapsingMergeTree -This engine differs from MergeTree in that it allows automatic deletion, or "collapsing" certain pairs of rows when merging. +*This engine is used specifically for Yandex.Metrica.* + +It differs from `MergeTree` in that it allows automatic deletion, or "collapsing" certain pairs of rows when merging. Yandex.Metrica has normal logs (such as hit logs) and change logs. Change logs are used for incrementally calculating statistics on data that is constantly changing. Examples are the log of session changes, or logs of changes to user histories. Sessions are constantly changing in Yandex.Metrica. For example, the number of hits per session increases. We refer to changes in any object as a pair (?old values, ?new values). Old values may be missing if the object was created. New values may be missing if the object was deleted. If the object was changed, but existed previously and was not deleted, both values are present. In the change log, one or two entries are made for each change. Each entry contains all the attributes that the object has, plus a special attribute for differentiating between the old and new values. When objects change, only the new entries are added to the change log, and the existing ones are not touched. @@ -11,11 +12,11 @@ This is the main concept that allows Yandex.Metrica to work in real time. CollapsingMergeTree accepts an additional parameter - the name of an Int8-type column that contains the row's "sign". Example: -.. code-block:: sql +```sql +CollapsingMergeTree(EventDate, (CounterID, EventDate, intHash32(UniqID), VisitID), 8192, Sign) +``` - CollapsingMergeTree(EventDate, (CounterID, EventDate, intHash32(UniqID), VisitID), 8192, Sign) - -Here, 'Sign' is a column containing -1 for "old" values and 1 for "new" values. +Here, `Sign` is a column containing -1 for "old" values and 1 for "new" values. When merging, each group of consecutive identical primary key values (columns for sorting data) is reduced to no more than one row with the column value 'sign_column = -1' (the "negative row") and no more than one row with the column value 'sign_column = 1' (the "positive row"). In other words, entries from the change log are collapsed. @@ -28,6 +29,8 @@ Thus, collapsing should not change the results of calculating statistics. Changes are gradually collapsed so that in the end only the last value of almost every object is left. Compared to MergeTree, the CollapsingMergeTree engine allows a multifold reduction of data volume. -There are several ways to get completely "collapsed" data from a CollapsingMergeTree table: - #. Write a query with GROUP BY and aggregate functions that accounts for the sign. For example, to calculate quantity, write 'sum(Sign)' instead of 'count()'. To calculate the sum of something, write 'sum(Sign * x)' instead of 'sum(x)', and so on, and also add 'HAVING sum(Sign) > 0'. Not all amounts can be calculated this way. For example, the aggregate functions 'min' and 'max' can't be rewritten. - #. If you must extract data without aggregation (for example, to check whether rows are present whose newest values match certain conditions), you can use the FINAL modifier for the FROM clause. This approach is significantly less efficient. +There are several ways to get completely "collapsed" data from a `CollapsingMergeTree` table: + +1. Write a query with GROUP BY and aggregate functions that accounts for the sign. For example, to calculate quantity, write 'sum(Sign)' instead of 'count()'. To calculate the sum of something, write 'sum(Sign * x)' instead of 'sum(x)', and so on, and also add 'HAVING sum(Sign) `>` 0'. Not all amounts can be calculated this way. For example, the aggregate functions 'min' and 'max' can't be rewritten. +2. If you must extract data without aggregation (for example, to check whether rows are present whose newest values match certain conditions), you can use the FINAL modifier for the FROM clause. This approach is significantly less efficient. + diff --git a/docs/en/table_engines/custom_partitioning_key.md b/docs/en/table_engines/custom_partitioning_key.md new file mode 100644 index 00000000000..eb056e35ebd --- /dev/null +++ b/docs/en/table_engines/custom_partitioning_key.md @@ -0,0 +1,47 @@ + + +# Custom partitioning key + +Starting with version 1.1.54310, you can create tables in the MergeTree family with any partition expression (not only partitioning by month). + +The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax: + +``` +ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...] +``` + +For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). Example: + +```sql +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) + PARTITION BY (toMonday(StartDate), EventType) + ORDER BY (CounterID, StartDate, intHash32(UserID)) + SAMPLE BY intHash32(UserID) +``` + +The traditional partitioning by month is expressed as `toYYYYMM(date_column)`. + +You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT). + +After this table is created, merge will only work for data parts that have the same value for the partition expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly. + +To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example: + +```sql +ALTER TABLE table DROP PARTITION (toMonday(today()), 1) +``` + +Deletes the partition for the current week with event type 1. The same is true for the OPTIMIZE query. To specify the only partition in a non-partitioned table, specify `PARTITION tuple()`. + +Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match). + +In the `system.parts` table, the `partition` column should specify the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
+Было: `20140317_20140323_2_2_0`
+(минимальная дата - максимальная дата - номер минимального блока - номер максимального блока - уровень).
+Стало: `201403_2_2_0`
+(ID партиции - номер минимального блока - номер максимального блока - уровень). + +The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`. + +There are more examples in the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql)and[`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). + diff --git a/docs/en/table_engines/distributed.rst b/docs/en/table_engines/distributed.md similarity index 63% rename from docs/en/table_engines/distributed.rst rename to docs/en/table_engines/distributed.md index c6edd1e9c76..2bff7a190f2 100644 --- a/docs/en/table_engines/distributed.rst +++ b/docs/en/table_engines/distributed.md @@ -1,66 +1,71 @@ -Distributed ------------ + -**The Distributed engine by itself does not store data**, but allows distributed query processing on multiple servers. +# Distributed + +**The Distributed engine does not store data itself**, but allows distributed query processing on multiple servers. Reading is automatically parallelized. During a read, the table indexes on remote servers are used, if there are any. The Distributed engine accepts parameters: the cluster name in the server's config file, the name of a remote database, the name of a remote table, and (optionally) a sharding key. Example: -.. code-block:: text +```text +Distributed(logs, default, hits[, sharding_key]) +``` - Distributed(logs, default, hits[, sharding_key]) +Data will be read from all servers in the 'logs' cluster, from the default.hits table located on every server in the cluster. +Data is not only read, but is partially processed on the remote servers (to the extent that this is possible). +For example, for a query with GROUP BY, data will be aggregated on remote servers, and the intermediate states of aggregate functions will be sent to the requestor server. Then data will be further aggregated. -Data will be read from all servers in the 'logs' cluster, from the 'default.hits' table located on every server in the cluster. Data is not only read, but is partially processed on the remote servers (to the extent that this is possible). For example, for a query with GROUP BY, data will be aggregated on remote servers, and the intermediate states of aggregate functions will be sent to the requestor server. Then data will be further aggregated. +Instead of the database name, you can use a constant expression that returns a string. For example: currentDatabase(). -Instead of the database name, you can use a constant expression that returns a string. For example, ``currentDatabase()``. - -logs - The cluster name in the server's config file. +logs – The cluster name in the server's config file. Clusters are set like this: -.. code-block:: xml +```xml + + + + + 1 + + false + + example01-01-1 + 9000 + + + example01-01-2 + 9000 + + + + 2 + false + + example01-02-1 + 9000 + + + example01-02-2 + 9000 + + + + +``` - - - - - 1 - - false - - example01-01-1 - 9000 - - - example01-01-2 - 9000 - - - - 2 - false - - example01-02-1 - 9000 - - - example01-02-2 - 9000 - - - - - -Here a cluster is defined with the name 'logs' that consists of two shards, each of which contains two replicas. Shards refer to the servers that contain different parts of the data (in order to read all the data, you must access all the shards). +Here a cluster is defined with the name 'logs' that consists of two shards, each of which contains two replicas. +Shards refer to the servers that contain different parts of the data (in order to read all the data, you must access all the shards). Replicas are duplicating servers (in order to read all the data, you can access the data on any one of the replicas). -For each server, there are several parameters: mandatory: ``'host'``, ``'port'``, and optional: ``'user'``, ``'password'``. - * ``host`` - address of remote server. May be specified as domain name or IPv4 or IPv6 address. If you specify domain, server will perform DNS lookup at startup, and result will be cached till server shutdown. If DNS request is failed, server won't start. If you are changing DNS records, restart the server for new records to take effect. - * ``port`` - TCP-port for interserver communication (tcp_port in configuration file, usually 9000). Don't get confused with http_port. - * ``user`` - user name to connect to remote server. By default user is 'default'. This user must have access rights to connect to remote server. Access rights are managed in users.xml configuration file. For additional info, consider "Access rights" section. - * ``password`` - password to log in to remote server, in plaintext. Default is empty string. +The parameters `host`, `port`, and optionally `user` and `password` are specified for each server: -When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) - see the 'load_balancing' setting. +: - `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. +- `port`– The TCP port for messenger activity ('tcp_port' in the config, usually set to 9000). Do not confuse it with http_port. +- `user`– Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section "Access rights". +- `password` – The password for connecting to a remote server (not masked). Default value: empty string. + +When specifying replicas, one of the available replicas will be selected for each of the shards when reading. You can configure the algorithm for load balancing (the preference for which replica to access) – see the 'load_balancing' setting. If the connection with the server is not established, there will be an attempt to connect with a short timeout. If the connection failed, the next replica will be selected, and so on for all the replicas. If the connection attempt failed for all the replicas, the attempt will be repeated the same way, several times. This works in favor of resiliency, but does not provide complete fault tolerance: a remote server might accept the connection, but might not work, or work poorly. @@ -74,12 +79,12 @@ The Distributed engine allows working with a cluster like a local server. Howeve There is no support for Distributed tables that look at other Distributed tables (except in cases when a Distributed table only has one shard). As an alternative, make the Distributed table look at the "final" tables. -The Distributed engine requires writing clusters to the config file. Clusters from config are updated on the fly, it does not require server restart. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table - use the 'remote' table function instead. See the section "Table functions". +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the 'remote' table function instead. See the section "Table functions". There are two methods for writing data to a cluster: First, you can define which servers to write which data to, and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". -This is the most flexible solution - you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. +This is the most flexible solution – you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution, since data can be written to different shards completely independently. Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across servers itself. @@ -93,19 +98,22 @@ If this parameter is set to 'true', the write operation selects the first health If it is set to 'false' (the default), data is written to all replicas. In essence, this means that the Distributed table replicates data itself. This is worse than using replicated tables, because the consistency of replicas is not checked, and over time they will contain slightly different data. -To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from 'prev_weight' to 'prev_weights + weight', where 'prev_weights' is the total weight of the shards with the smallest number, and 'weight' is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range [0, 9), and to the second for the remainders from the range [10, 19). +To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from 'prev_weight' to 'prev_weights + weight', where 'prev_weights' is the total weight of the shards with the smallest number, and 'weight' is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range \[0, 9), and to the second for the remainders from the range \[10, 19). The sharding expression can be any expression from constants and table columns that returns an integer. For example, you can use the expression 'rand()' for random distribution of data, or 'UserID' for distribution by the remainder from dividing the user's ID (then the data of a single user will reside on a single shard, which simplifies running IN and JOIN by users). If one of the columns is not distributed evenly enough, you can wrap it in a hash function: intHash64(UserID). A simple remainder from division is a limited solution for sharding and isn't always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area, rather than using entries in Distributed tables. -SELECT queries are sent to all the shards, and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight - the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +When using replicated tables, data can be resharded; see the section "Resharding". But in many cases you are better off without it. SELECT queries are sent to all the shards, and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. You should be concerned about the sharding scheme in the following cases: + - Queries are used that require joining data (IN or JOIN) by a specific key. If data is sharded by this key, you can use local IN or JOIN instead of GLOBAL IN or GLOBAL JOIN, which is much more efficient. - A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we've done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into "layers", where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. -Data is written asynchronously. For an INSERT to a Distributed table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: -/var/lib/clickhouse/data/database/table/. +Data is written asynchronously. For an INSERT to a Distributed table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: /var/lib/clickhouse/data/database/table/. If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the 'broken' subdirectory and no longer used. + +When the max_parallel_replicas option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section "Settings, max_parallel_replicas". + diff --git a/docs/en/table_engines/external_data.md b/docs/en/table_engines/external_data.md new file mode 100644 index 00000000000..90dc4bded04 --- /dev/null +++ b/docs/en/table_engines/external_data.md @@ -0,0 +1,60 @@ +# External data for query processing + +ClickHouse allows sending a server the data that is needed for processing a query, together with a SELECT query. This data is put in a temporary table (see the section "Temporary tables") and can be used in the query (for example, in IN operators). + +For example, if you have a text file with important user identifiers, you can upload it to the server along with a query that uses filtration by this list. + +If you need to run more than one query with a large volume of external data, don't use this feature. It is better to upload the data to the DB ahead of time. + +External data can be uploaded using the command-line client (in non-interactive mode), or using the HTTP interface. + +In the command-line client, you can specify a parameters section in the format + +```bash +--external --file=... [--name=...] [--format=...] [--types=...|--structure=...] +``` + +You may have multiple sections like this, for the number of tables being transmitted. + +**--external** – Marks the beginning of a clause. +**--file** – Path to the file with the table dump, or -, which refers to stdin. +Only a single table can be retrieved from stdin. + +The following parameters are optional: **--name**– Name of the table. If omitted, _data is used. +**--format** – Data format in the file. If omitted, TabSeparated is used. + +One of the following parameters is required:**--types** – A list of comma-separated column types. For example: `UInt64,String`. The columns will be named _1, _2, ... +**--structure**– The table structure in the format`UserID UInt64`, `URL String`. Defines the column names and types. + +The files specified in 'file' will be parsed by the format specified in 'format', using the data types specified in 'types' or 'structure'. The table will be uploaded to the server and accessible there as a temporary table with the name in 'name'. + +Examples: + +```bash +echo -ne "1\n2\n3\n" | clickhouse-client --query="SELECT count() FROM test.visits WHERE TraficSourceID IN _data" --external --file=- --types=Int8 +849897 +cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, count() AS c FROM passwd GROUP BY shell ORDER BY c DESC" --external --file=- --name=passwd --structure='login String, unused String, uid UInt16, gid UInt16, comment String, home String, shell String' +/bin/sh 20 +/bin/false 5 +/bin/bash 4 +/usr/sbin/nologin 1 +/bin/sync 1 +``` + +When using the HTTP interface, external data is passed in the multipart/form-data format. Each table is transmitted as a separate file. The table name is taken from the file name. The 'query_string' is passed the parameters 'name_format', 'name_types', and 'name_structure', where 'name' is the name of the table that these parameters correspond to. The meaning of the parameters is the same as when using the command-line client. + +Example: + +```bash +cat /etc/passwd | sed 's/:/\t/g' > passwd.tsv + +curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+count()+AS+c+FROM+passwd+GROUP+BY+shell+ORDER+BY+c+DESC&passwd_structure=login+String,+unused+String,+uid+UInt16,+gid+UInt16,+comment+String,+home+String,+shell+String' +/bin/sh 20 +/bin/false 5 +/bin/bash 4 +/usr/sbin/nologin 1 +/bin/sync 1 +``` + +For distributed query processing, the temporary tables are sent to all the remote servers. + diff --git a/docs/en/table_engines/external_data.rst b/docs/en/table_engines/external_data.rst deleted file mode 100644 index 253f78e16fe..00000000000 --- a/docs/en/table_engines/external_data.rst +++ /dev/null @@ -1,62 +0,0 @@ -External data for query processing -================================== - -ClickHouse allows sending a server the data that is needed for processing a query, together with a SELECT query. This data is put in a temporary table (see the section "Temporary tables") and can be used in the query (for example, in IN operators). - -For example, if you have a text file with important user identifiers, you can upload it to the server along with a query that uses filtration by this list. - -If you need to run more than one query with a large volume of external data, don't use this feature. It is better to upload the data to the DB ahead of time. - -External data can be uploaded using the command-line client (in non-interactive mode), or using the HTTP interface. - -In the command-line client, you can specify a parameters section in the format - -.. code-block:: bash - - --external --file=... [--name=...] [--format=...] [--types=...|--structure=...] - -You may have multiple sections like this, for the number of tables being transmitted. - -**--external** - Marks the beginning of the section. -**--file** - Path to the file with the table dump, or ``-``, which refers to stdin -Only a single table can be retrieved from stdin. - -The following parameters are optional: -**--name** - Name of the table. If omitted, ``_data`` is used. -**--format** - Data format in the file. If omitted, ``TabSeparated`` is used. - -One of the following parameters is required: -**--types** - A comma-separated list of column types. For example, ``UInt64,String``. Columns will be named ``_1``, ``_2``, ... -**--structure** - Table structure, in the format ``UserID UInt64, URL String``. Defines the column names and types. - -The files specified in ``file`` will be parsed by the format specified in ``format``, using the data types specified in ``types`` or ``structure``. The table will be uploaded to the server and accessible there as a temporary table with the name ``name``. - -Examples: - -.. code-block:: bash - - echo -ne "1\n2\n3\n" | clickhouse-client --query="SELECT count() FROM test.visits WHERE TraficSourceID IN _data" --external --file=- --types=Int8 - 849897 - cat /etc/passwd | sed 's/:/\t/g' | clickhouse-client --query="SELECT shell, count() AS c FROM passwd GROUP BY shell ORDER BY c DESC" --external --file=- --name=passwd --structure='login String, unused String, uid UInt16, gid UInt16, comment String, home String, shell String' - /bin/sh 20 - /bin/false 5 - /bin/bash 4 - /usr/sbin/nologin 1 - /bin/sync 1 - -When using the HTTP interface, external data is passed in the multipart/form-data format. Each table is transmitted as a separate file. The table name is taken from the file name. The 'query_string' passes the parameters 'name_format', 'name_types', and 'name_structure', where name is the name of the table that these parameters correspond to. The meaning of the parameters is the same as when using the command-line client. - -Example: - -.. code-block:: bash - - cat /etc/passwd | sed 's/:/\t/g' > passwd.tsv - - curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+count()+AS+c+FROM+passwd+GROUP+BY+shell+ORDER+BY+c+DESC&passwd_structure=login+String,+unused+String,+uid+UInt16,+gid+UInt16,+comment+String,+home+String,+shell+String' - /bin/sh 20 - /bin/false 5 - /bin/bash 4 - /usr/sbin/nologin 1 - /bin/sync 1 - -For distributed query processing, the temporary tables are sent to all the remote servers. diff --git a/docs/en/table_engines/file.md b/docs/en/table_engines/file.md new file mode 100644 index 00000000000..a6d5f9eba43 --- /dev/null +++ b/docs/en/table_engines/file.md @@ -0,0 +1,4 @@ +# File(InputFormat) + +The data source is a file that stores data in one of the supported input formats (TabSeparated, Native, etc.). + diff --git a/docs/en/table_engines/file.rst b/docs/en/table_engines/file.rst deleted file mode 100644 index 55411655b32..00000000000 --- a/docs/en/table_engines/file.rst +++ /dev/null @@ -1,4 +0,0 @@ -File(InputFormat) ------------------ - -The data source is a file that stores data in one of the supported input formats (TabSeparated, Native, etc.) ... diff --git a/docs/en/table_engines/graphitemergetree.md b/docs/en/table_engines/graphitemergetree.md new file mode 100644 index 00000000000..6452377ac15 --- /dev/null +++ b/docs/en/table_engines/graphitemergetree.md @@ -0,0 +1,86 @@ + + +# GraphiteMergeTree + +This engine is designed for rollup (thinning and aggregating/averaging) [ Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. + +Graphite stores full data in ClickHouse, and data can be retrieved in the following ways: + +- Without thinning. + + Using the [MergeTree](mergetree.md#table_engines-mergetree) engine. + +- With thinning. + + Using the `GraphiteMergeTree` engine. + +The engine inherits properties from MergeTree. The settings for thinning data are defined by the [graphite_rollup](../operations/server_settings/settings.md#server_settings-graphite_rollup) parameter in the server configuration. + +## Using the engine + +The Graphite data table must contain the following fields at minimum: + +- `Path` – The metric name (Graphite sensor). +- `Time` – The time for measuring the metric. +- `Value` – The value of the metric at the time set in Time. +- `Version` – Determines which value of the metric with the same Path and Time will remain in the database. + +Rollup pattern: + +```text +pattern + regexp + function + age -> precision + ... +pattern + ... +default + function + age -> precision + ... +``` + +When processing a record, ClickHouse will check the rules in the `pattern`clause. If the metric name matches the `regexp`, the rules from `pattern` are applied; otherwise, the rules from `default` are used. + +Fields in the pattern. + +- `age` – The minimum age of the data in seconds. +- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. +- `precision`– How precisely to define the age of the data in seconds. +- `regexp`– A pattern for the metric name. + +Example of settings: + +```xml + + + click_cost + any + + 0 + 5 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + diff --git a/docs/en/table_engines/index.md b/docs/en/table_engines/index.md new file mode 100644 index 00000000000..f8454e5e155 --- /dev/null +++ b/docs/en/table_engines/index.md @@ -0,0 +1,21 @@ +# Table engines + +The table engine (type of table) determines: + +- How and where data is stored: where to write it to, and where to read it from. +- Which queries are supported, and how. +- Concurrent data access. +- Use of indexes, if present. +- Whether multithreaded request execution is possible. +- Data replication. +- When reading data, the engine is only required to extract the necessary set of columns.However, in some cases, the query may be partially processed inside the table engine. + +Note that for most serious tasks, you should use engines from the MergeTree family. + +```eval_rst +.. toctree:: + :glob: + + * +``` + diff --git a/docs/en/table_engines/index.rst b/docs/en/table_engines/index.rst deleted file mode 100644 index c8e972a89f6..00000000000 --- a/docs/en/table_engines/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -Table engines -============= - -The table engine (type of table) determines: - - How and where data is stored - where to write it to, and where to read it from. - - Which queries are supported, and how. - - Concurrent data access. - - Use of indexes, if present. - - Whether multithreaded request execution is possible. - - Data replication. - - When reading data, the engine is only required to extract the necessary set of columns. However, in some cases, the query may be partially processed inside the table engine. - -Note that for most serious tasks, you should use engines from the MergeTree family. - -.. toctree:: - :glob: - - * diff --git a/docs/en/table_engines/join.rst b/docs/en/table_engines/join.md similarity index 57% rename from docs/en/table_engines/join.rst rename to docs/en/table_engines/join.md index 85ec9cfd7f0..1aff89ca887 100644 --- a/docs/en/table_engines/join.rst +++ b/docs/en/table_engines/join.md @@ -1,16 +1,17 @@ -Join ----- +# Join A prepared data structure for JOIN that is always located in RAM. -.. code-block:: text +```text +Join(ANY|ALL, LEFT|INNER, k1[, k2, ...]) +``` - Join(ANY|ALL, LEFT|INNER, k1[, k2, ...]) - -Engine parameters: ``ANY``|``ALL`` - strictness, and ``LEFT``|``INNER`` - the type. These parameters are set without quotes and must match the JOIN that the table will be used for. k1, k2, ... are the key columns from the USING clause that the join will be made on. +Engine parameters: `ANY|ALL` – strictness; `LEFT|INNER` – type. +These parameters are set without quotes and must match the JOIN that the table will be used for. k1, k2, ... are the key columns from the USING clause that the join will be made on. The table can't be used for GLOBAL JOINs. You can use INSERT to add data to the table, similar to the Set engine. For ANY, data for duplicated keys will be ignored. For ALL, it will be counted. You can't perform SELECT directly from the table. The only way to retrieve data is to use it as the "right-hand" table for JOIN. Storing data on the disk is the same as for the Set engine. + diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md new file mode 100644 index 00000000000..02e4de01dd7 --- /dev/null +++ b/docs/en/table_engines/kafka.md @@ -0,0 +1,79 @@ +# Kafka + +The engine works with [Apache Kafka](http://kafka.apache.org/). + +Kafka lets you: + +- Publish or subscribe to data flows. +- Organize fault-tolerant storage. +- Process streams as they become available. + +``` +Kafka(broker_list, topic_list, group_name, format[, schema]) +``` + +Parameters: + +- `broker_list` – A comma-separated list of brokers (`localhost:9092`). +- `topic_list` – A list of Kafka topics (`my_topic`). +- `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `--format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. +- `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root ` schema.capnp:Message` object. + +Example: + +```sql +CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + SELECT * FROM queue LIMIT 5; +``` + +The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. + +Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. For more information, see [http://kafka.apache.org/intro](http://kafka.apache.org/intro). + +`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using materialized views. For this purpose, the following was done: + +1. Use the engine to create a Kafka consumer and consider it a data stream. +2. Create a table with the desired structure. +3. Create a materialized view that converts data from the engine and puts it into a previously created table. + +When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from Kafka and convert them to the required format using `SELECT` + +Example: + +```sql +CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + CREATE TABLE daily ( + day Date, + level String, + total UInt64 + ) ENGINE = SummingMergeTree(day, (day, level), 8192); + + CREATE MATERIALIZED VIEW consumer TO daily + AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total + FROM queue GROUP BY day, level; + +SELECT level, sum(total) FROM daily GROUP BY level; +``` + +To improve performance, received messages are grouped into blocks the size of [max_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). If the block wasn't formed within [ stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block. + +To stop receiving topic data or to change the conversion logic, detach the materialized view: + +``` +DETACH TABLE consumer; +ATTACH MATERIALIZED VIEW consumer; +``` + +If you want to change the target table by using ` ALTER`materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view. + diff --git a/docs/en/table_engines/kafka.rst b/docs/en/table_engines/kafka.rst deleted file mode 100644 index d31fdaf4491..00000000000 --- a/docs/en/table_engines/kafka.rst +++ /dev/null @@ -1,111 +0,0 @@ -Kafka ------ - -A table engine backed by Apache Kafka, a streaming platform having three key capabilities: - -1. It lets you publish and subscribe to streams of records. In this respect it is similar to a message queue or enterprise messaging system. -2. It lets you store streams of records in a fault-tolerant way. -3. It lets you process streams of records as they occur. - -.. code-block:: text - - Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers]) - -Engine parameters: - -broker_list - A comma-separated list of brokers (``localhost:9092``). - -topic_list - List of Kafka topics to consume (``my_topic``). - -group_name - Kafka consumer group name (``group1``). Read offsets are tracked for each consumer group, if you want to consume messages exactly once across cluster, you should use the same group name. - -format - Name of the format used to deserialize messages. It accepts the same values as the ``FORMAT`` SQL statement, for example ``JSONEachRow``. - -schema - Optional schema value for formats that require a schema to interpret consumed messages, for example Cap'n Proto format requires - a path to schema file and root object - ``schema:Message``. Self-describing formats such as JSON don't require any schema. - -num_consumers - Number of created consumers per engine. By default ``1``. Create more consumers if the throughput of a single consumer is insufficient. The total number of consumers shouldn't exceed the number of partitions in given topic, as there can be at most 1 consumers assigned to any single partition. - -Example: - -.. code-block:: sql - - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - SELECT * FROM queue LIMIT 5; - -The consumed messages are tracked automatically in the background, so each message will be read exactly once in a single consumer group. If you want to consume the same set of messages twice, you can create a copy of the table with a different ``group_name``. The consumer group is elastic and synchronised across the cluster. For example, if you have 10 topic/partitions and 5 instances of the table across cluster, it will automatically assign 2 topic/partitions per instace. If you detach a Kafka engine table (or create new), it will rebalance topic/partition assignments automatically. See `Kafka Introduction `_ for more information about how this works. - -Reading messages using SELECT is not very useful (except for troubleshooting), because each message can be read only once. -The table engine is typically used to build real-time ingestion pipelines using MATERIALIZED VIEW. It works like this: - -1. You create a Kafka consumer with a Kafka engine. This is the data stream. -2. You create an arbitrary table with the desired data schema. -3. You create a MATERIALIZED VIEW that transforms the data from Kafka, and materializes it into your desired table. - - -When a MATERIALIZED VIEW is attached to a Kafka table engine, it will start automatically consuming messages in the background, and push them into the attached views. This allows you to continuously ingest messages from Kafka and transform them using the SELECT statement to describe transformation. - -Example: - -.. code-block:: sql - - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - CREATE TABLE daily ( - day Date, - level String, - total UInt64 - ) ENGINE = SummingMergeTree(day, (day, level), 8192); - - CREATE MATERIALIZED VIEW consumer TO daily - AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total - FROM queue GROUP BY day, level; - - SELECT level, sum(total) FROM daily GROUP BY level; - -The messages are streamed into the attached view immediately in the same way a continuous stream of INSERT statement would. To improve performance, consumed messages are squashed into batches of ``max_insert_block_size``. If the message batch cannot be completed within ``stream_flush_interval_ms`` period (by default 7500ms), it will be flushed to ensure time bounded insertion time. - -In order to stop topic consumption, or alter the transformation logc, you simply detach the MATERIALIZED VIEW: - -.. code-block:: sql - - DETACH TABLE consumer; - ATTACH MATERIALIZED VIEW consumer; - -Note: When you're performing ALTERs on target table, it's recommended to detach materializing views to prevent a mismatch between the current schema and the result of MATERIALIZED VIEWS. - -Configuration -~~~~~~~~~~~~~ - -Similarly to GraphiteMergeTree, Kafka engine supports extended configuration through the ClickHouse config file. There are two configuration keys you can use - global, and per-topic. The global configuration is applied first, then per-topic configuration (if exists). - -.. code-block:: xml - - - - cgrp - smallest - - - - - 250 - 100000 - - -See `librdkafka configuration reference `_ for the list of possible configuration options. Use underscores instead of dots in the ClickHouse configuration, for example ``check.crcs=true`` would correspond to ``true``. diff --git a/docs/en/table_engines/log.rst b/docs/en/table_engines/log.md similarity index 54% rename from docs/en/table_engines/log.rst rename to docs/en/table_engines/log.md index 80defc21648..406f94bd0d0 100644 --- a/docs/en/table_engines/log.rst +++ b/docs/en/table_engines/log.md @@ -1,5 +1,6 @@ -Log ---- +# Log -Log differs from TinyLog in that a small file of "marks" resides with the column files. These marks are written on every data block and contain offsets - where to start reading the file in order to skip the specified number of rows. This makes it possible to read table data in multiple threads. For concurrent data access, the read operations can be performed simultaneously, while write operations block reads and each other. +Log differs from TinyLog in that a small file of "marks" resides with the column files. These marks are written on every data block and contain offsets that indicate where to start reading the file in order to skip the specified number of rows. This makes it possible to read table data in multiple threads. +For concurrent data access, the read operations can be performed simultaneously, while write operations block reads and each other. The Log engine does not support indexes. Similarly, if writing to a table failed, the table is broken, and reading from it returns an error. The Log engine is appropriate for temporary data, write-once tables, and for testing or demonstration purposes. + diff --git a/docs/en/table_engines/materializedview.md b/docs/en/table_engines/materializedview.md new file mode 100644 index 00000000000..3707b27078e --- /dev/null +++ b/docs/en/table_engines/materializedview.md @@ -0,0 +1,4 @@ +# MaterializedView + +Used for implementing materialized views (for more information, see `CREATE MATERIALIZED VIEW`). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. + diff --git a/docs/en/table_engines/materializedview.rst b/docs/en/table_engines/materializedview.rst deleted file mode 100644 index 08469af53ed..00000000000 --- a/docs/en/table_engines/materializedview.rst +++ /dev/null @@ -1,4 +0,0 @@ -MaterializedView ----------------- - -Used for implementing materialized views (for more information, see ``CREATE MATERIALIZED VIEW``). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. diff --git a/docs/en/table_engines/memory.rst b/docs/en/table_engines/memory.md similarity index 98% rename from docs/en/table_engines/memory.rst rename to docs/en/table_engines/memory.md index 07b01933615..113e4aa9f11 100644 --- a/docs/en/table_engines/memory.rst +++ b/docs/en/table_engines/memory.md @@ -1,5 +1,4 @@ -Memory ------- +# Memory The Memory engine stores data in RAM, in uncompressed form. Data is stored in exactly the same form as it is received when read. In other words, reading from this table is completely free. Concurrent data access is synchronized. Locks are short: read and write operations don't block each other. @@ -9,3 +8,4 @@ When restarting a server, data disappears from the table and the table becomes e Normally, using this table engine is not justified. However, it can be used for tests, and for tasks where maximum speed is required on a relatively small number of rows (up to approximately 100,000,000). The Memory engine is used by the system for temporary tables with external query data (see the section "External data for processing a query"), and for implementing GLOBAL IN (see the section "IN operators"). + diff --git a/docs/en/table_engines/merge.rst b/docs/en/table_engines/merge.md similarity index 53% rename from docs/en/table_engines/merge.rst rename to docs/en/table_engines/merge.md index 6f28d730725..10424aa3f10 100644 --- a/docs/en/table_engines/merge.rst +++ b/docs/en/table_engines/merge.md @@ -1,37 +1,38 @@ -Merge ------ +# Merge -The Merge engine (not to be confused with MergeTree) does not store data itself, but allows reading from any number of other tables simultaneously. +The Merge engine (not to be confused with `MergeTree`) does not store data itself, but allows reading from any number of other tables simultaneously. Reading is automatically parallelized. Writing to a table is not supported. When reading, the indexes of tables that are actually being read are used, if they exist. The Merge engine accepts parameters: the database name and a regular expression for tables. Example: -.. code-block:: text +```text +Merge(hits, '^WatchLog') +``` - Merge(hits, '^WatchLog') +- Data will be read from the tables in the 'hits' database that have names that match the regular expression '`^WatchLog`'. -- Data will be read from the tables in the 'hits' database with names that match the regex ``'^WatchLog'``. +Instead of the database name, you can use a constant expression that returns a string. For example, `currentDatabase()`. -Instead of the database name, you can use a constant expression that returns a string. For example, ``currentDatabase()``. - -Regular expressions are re2 (similar to PCRE), case-sensitive. See the notes about escaping symbols in regular expressions in the "match" section. +Regular expressions are re2 (similar to PCRE), case-sensitive. +See the notes about escaping symbols in regular expressions in the "match" section. When selecting tables to read, the Merge table itself will not be selected, even if it matches the regex. This is to avoid loops. It is possible to create two Merge tables that will endlessly try to read each others' data. But don't do this. The typical way to use the Merge engine is for working with a large number of TinyLog tables as if with a single table. -Virtual columns -~~~~~~~~~~~~~~~ +## Virtual columns Virtual columns are columns that are provided by the table engine, regardless of the table definition. In other words, these columns are not specified in CREATE TABLE, but they are accessible for SELECT. Virtual columns differ from normal columns in the following ways: - - They are not specified in table definitions. - - Data can't be added to them with ``INSERT``. - - When using ``INSERT`` without specifying the list of columns, virtual columns are ignored. - - They are not selected when using the asterisk (``SELECT *``). - - Virtual columns are not shown in ``SHOW CREATE TABLE`` and ``DESC TABLE`` queries. -A Merge table contains the virtual column **_table** of the String type. (If the table already has a '_table' column, the virtual column is named '_table1', and if it already has '_table1', it is named '_table2', and so on.) It contains the name of the table that data was read from. +- They are not specified in table definitions. +- Data can't be added to them with INSERT. +- When using INSERT without specifying the list of columns, virtual columns are ignored. +- They are not selected when using the asterisk (`SELECT *`). +- Virtual columns are not shown in `SHOW CREATE TABLE` and `DESC TABLE` queries. + +A Merge type table contains a virtual _table column with the String type. (If the table already has a _table column, the virtual column is named _table1, and if it already has _table1, it is named _table2, and so on.) It contains the name of the table that data was read from. If the WHERE or PREWHERE clause contains conditions for the '_table' column that do not depend on other table columns (as one of the conjunction elements, or as an entire expression), these conditions are used as an index. The conditions are performed on a data set of table names to read data from, and the read operation will be performed from only those tables that the condition was triggered on. + diff --git a/docs/en/table_engines/mergetree.rst b/docs/en/table_engines/mergetree.md similarity index 71% rename from docs/en/table_engines/mergetree.rst rename to docs/en/table_engines/mergetree.md index aa37900d833..71197f21b34 100644 --- a/docs/en/table_engines/mergetree.rst +++ b/docs/en/table_engines/mergetree.md @@ -1,23 +1,23 @@ -MergeTree ---------- + + +# MergeTree The MergeTree engine supports an index by primary key and by date, and provides the possibility to update data in real time. This is the most advanced table engine in ClickHouse. Don't confuse it with the Merge engine. -The engine accepts parameters: the name of a Date type column containing the date, a sampling expression (optional), a tuple that defines the table's primary key, and the index granularity. -Example: +The engine accepts parameters: the name of a Date type column containing the date, a sampling expression (optional), a tuple that defines the table's primary key, and the index granularity. Example: Example without sampling support: -.. code-block:: text - - MergeTree(EventDate, (CounterID, EventDate), 8192) +```text +MergeTree(EventDate, (CounterID, EventDate), 8192) +``` Example with sampling support: -.. code-block:: text - - MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) +```text +MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) +``` A MergeTree type table must have a separate column containing the date. In this example, it is the 'EventDate' column. The type of the date column must be 'Date' (not 'DateTime'). @@ -37,23 +37,26 @@ For each part, an index file is also written. The index file contains the primar For columns, "marks" are also written to each 'index_granularity' row so that data can be read in a specific range. -When reading from a table, the SELECT query is analyzed for whether indexes can be used. An index can be used if the WHERE or PREWHERE clause has an expression (as one of the conjunction elements, or entirely) that represents an equality or inequality comparison operation, or if it has IN above columns that are in the primary key or date, or Boolean operators over them. +When reading from a table, the SELECT query is analyzed for whether indexes can be used. +An index can be used if the WHERE or PREWHERE clause has an expression (as one of the conjunction elements, or entirely) that represents an equality or inequality comparison operation, or if it has IN above columns that are in the primary key or date, or Boolean operators over them. Thus, it is possible to quickly run queries on one or many ranges of the primary key. In the example given, queries will work quickly for a specific counter, for a specific counter and range of dates, for a specific counter and date, for multiple counters and a range of dates, and so on. -.. code-block:: sql - - SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 - SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) - SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +```sql +SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 +SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) +SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +``` All of these cases will use the index by date and by primary key. The index is used even for complex expressions. Reading from the table is organized so that using the index can't be slower than a full scan. In this example, the index can't be used: -.. code-block:: sql +```sql +SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' +``` - SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' +To check whether ClickHouse can use the index when executing the query, use the settings [ force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date) and [ force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key). The index by date only allows reading those parts that contain dates from the desired range. However, a data part may contain data for many dates (up to an entire month), while within a single part the data is ordered by the primary key, which might not contain the date as the first column. Because of this, using a query with only a date condition that does not specify the primary key prefix will cause more data to be read than for a single date. @@ -61,8 +64,9 @@ For concurrent table access, we use multi-versioning. In other words, when a tab Reading from a table is automatically parallelized. -The OPTIMIZE query is supported, which calls an extra merge step. +The `OPTIMIZE` query is supported, which calls an extra merge step. -You can use a single large table and continually add data to it in small chunks - this is what MergeTree is intended for. +You can use a single large table and continually add data to it in small chunks – this is what MergeTree is intended for. Data replication is possible for all types of tables in the MergeTree family (see the section "Data replication"). + diff --git a/docs/en/table_engines/null.rst b/docs/en/table_engines/null.md similarity index 68% rename from docs/en/table_engines/null.rst rename to docs/en/table_engines/null.md index ee3260f8bf2..49850ec9864 100644 --- a/docs/en/table_engines/null.rst +++ b/docs/en/table_engines/null.md @@ -1,6 +1,6 @@ -Null ----- +# Null When writing to a Null table, data is ignored. When reading from a Null table, the response is empty. -However, you can create a materialized view on a Null table, so the data written to the table will end up in the view. +However, you can create a materialized view on a Null table. So the data written to the table will end up in the view. + diff --git a/docs/en/table_engines/replacingmergetree.md b/docs/en/table_engines/replacingmergetree.md new file mode 100644 index 00000000000..66332d44356 --- /dev/null +++ b/docs/en/table_engines/replacingmergetree.md @@ -0,0 +1,18 @@ +# ReplacingMergeTree + +This engine table differs from `MergeTree` in that it removes duplicate entries with the same primary key value. + +The last optional parameter for the table engine is the "version" column. When merging, it reduces all rows with the same primary key value to just one row. If the version column is specified, it leaves the row with the highest version; otherwise, it leaves the last row. + +The version column must have a type from the `UInt` family, `Date`, or `DateTime`. + +```sql +ReplacingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192, ver) +``` + +Note that data is only deduplicated during merges. Merging occurs in the background at an unknown time, so you can't plan for it. Some of the data may remain unprocessed. Although you can run an unscheduled merge using the OPTIMIZE query, don't count on using it, because the OPTIMIZE query will read and write a large amount of data. + +Thus, `ReplacingMergeTree` is suitable for clearing out duplicate data in the background in order to save space, but it doesn't guarantee the absence of duplicates. + +*This engine is not used in Yandex.Metrica, but it has been applied in other Yandex projects.* + diff --git a/docs/en/table_engines/replacingmergetree.rst b/docs/en/table_engines/replacingmergetree.rst deleted file mode 100644 index 3635cb99d58..00000000000 --- a/docs/en/table_engines/replacingmergetree.rst +++ /dev/null @@ -1,19 +0,0 @@ -ReplacingMergeTree ------------------- - -This engine differs from ``MergeTree`` in that it can deduplicate data by primary key while merging. - -For ReplacingMergeTree mode, last parameter is optional name of 'version' column. While merging, for all rows with same primary key, only one row is selected: last row, if version column was not specified, or last row with maximum version value, if specified. - -Version column must have type of UInt family or ``Date`` or ``DateTime``. - -.. code-block:: sql - - ReplacingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192, ver) - -Please note, that data is deduplicated only while merging process. Merges are processed in background. Exact time of merge is unspecified and you could not rely on it. Some part of data could be not merged at all. While you could trigger extra merge with OPTIMIZE query, it is not recommended, as OPTIMIZE will read and write vast amount of data. - -This table engine is suitable for background removal of duplicate data to save space, but not suitable to guarantee of deduplication. - -*Developed for special purposes of not Yandex.Metrica department.* - diff --git a/docs/en/table_engines/replication.rst b/docs/en/table_engines/replication.md similarity index 62% rename from docs/en/table_engines/replication.rst rename to docs/en/table_engines/replication.md index efdbb6dab84..0ebd5ccfc51 100644 --- a/docs/en/table_engines/replication.rst +++ b/docs/en/table_engines/replication.md @@ -1,23 +1,14 @@ -Data replication ----------------- + -ReplicatedMergeTree -~~~~~~~~~~~~~~~~~~~ +# Data replication -ReplicatedCollapsingMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## ReplicatedMergeTree -ReplicatedAggregatingMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## ReplicatedCollapsingMergeTree -ReplicatedSummingMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~ +## ReplicatedAggregatingMergeTree -ReplicatedReplacingMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -ReplicatedGraphiteMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## ReplicatedSummingMergeTree Replication is only supported for tables in the MergeTree family. Replication works at the level of an individual table, not the entire server. A server can store both replicated and non-replicated tables at the same time. @@ -28,42 +19,42 @@ Replication is not related to sharding in any way. Replication works independent Replication is an optional feature. To use replication, set the addresses of the ZooKeeper cluster in the config file. Example: -.. code-block:: xml +```xml + + + example1 + 2181 + + + example2 + 2181 + + + example3 + 2181 + + +``` - - - example1 - 2181 - - - example2 - 2181 - - - example3 - 2181 - - +**Use ZooKeeper version 3.4.5 or later.** For example, the version in the Ubuntu Precise package is too old. -**Use ZooKeeper version 3.4.5 or later** For example, the version in the Ubuntu Precise package is too old. - -You can specify any existing ZooKeeper cluster - the system will use a directory on it for its own data (the directory is specified when creating a replicatable table). +You can specify any existing ZooKeeper cluster and the system will use a directory on it for its own data (the directory is specified when creating a replicatable table). If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only. -ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries - they work just as fast as for non-replicated tables. +ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries). For each INSERT query (more precisely, for each inserted block of data; the INSERT query contains a single block, or per block for every max_insert_block_size = 1048576 rows), approximately ten entries are made in ZooKeeper in several transactions. This leads to slightly longer latencies for INSERT compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one INSERT per second, it doesn't create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred INSERTs per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. For very large clusters, you can use different ZooKeeper clusters for different shards. However, this hasn't proven necessary on the Yandex.Metrica cluster (approximately 300 servers). -Replication is asynchronous and multi-master. INSERT queries (as well as ALTER) can be sent to any available server. Data is inserted on this server, then sent to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If a part of the replicas is not available, the data on them is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. +Replication is asynchronous and multi-master. INSERT queries (as well as ALTER) can be sent to any available server. Data is inserted on this server, then sent to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If part of the replicas are not available, the data on them is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. There are no quorum writes. You can't write data with confirmation that it was received by more than one replica. If you write a batch of data to one replica and the server with this data ceases to exist before the data has time to get to the other replicas, this data will be lost. Each block of data is written atomically. The INSERT query is divided into blocks up to max_insert_block_size = 1048576 rows. In other words, if the INSERT query has less than 1048576 rows, it is made atomically. -Blocks of data are deduplicated. For multiple writes of the same data block (data blocks of the same size containing the same rows in the same order), the block is only written once. The reason for this is in case of network failures when the client application doesn't know if the data was written to the DB, so the INSERT query can simply be repeated. It doesn't matter which replica INSERTs were sent to with identical data - INSERTs are idempotent. This only works for the last 100 blocks inserted in a table. +Data blocks are deduplicated. For multiple writes of the same data block (data blocks of the same size containing the same rows in the same order), the block is only written once. The reason for this is in case of network failures when the client application doesn't know if the data was written to the DB, so the INSERT query can simply be repeated. It doesn't matter which replica INSERTs were sent to with identical data. INSERTs are idempotent. This only works for the last 100 blocks inserted in a table. During replication, only the source data to insert is transferred over the network. Further data transformation (merging) is coordinated and performed on all the replicas in the same way. This minimizes network usage, which means that replication works well when replicas reside in different datacenters. (Note that duplicating data in different datacenters is the main goal of replication.) @@ -71,37 +62,38 @@ You can have any number of replicas of the same data. Yandex.Metrica uses double The system monitors data synchronicity on replicas and is able to recover after a failure. Failover is automatic (for small differences in data) or semi-automatic (when data differs too much, which may indicate a configuration error). -Creating replicated tables -~~~~~~~~~~~~~~~~~~~~~~~~~~ + -The ``'Replicated'`` prefix is added to the table engine name. For example, ``ReplicatedMergeTree``. +## Creating replicated tables -Two parameters are also added in the beginning of the parameters list - the path to the table in ZooKeeper, and the replica name in ZooKeeper. +The `Replicated` prefix is added to the table engine name. For example:`ReplicatedMergeTree`. + +Two parameters are also added in the beginning of the parameters list – the path to the table in ZooKeeper, and the replica name in ZooKeeper. Example: -.. code-block:: text - - ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +```text +ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +``` As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the 'macros' section of the config file. Example: -.. code-block:: xml - - - 05 - 02 - example05-02-1.yandex.ru - +```xml + + 05 + 02 + example05-02-1.yandex.ru + +``` The path to the table in ZooKeeper should be unique for each replicated table. Tables on different shards should have different paths. In this case, the path consists of the following parts: -``/clickhouse/tables/`` - is the common prefix. We recommend using exactly this one. +`/clickhouse/tables/` is the common prefix. We recommend using exactly this one. -``{layer}-{shard}`` - is the shard identifier. In this example it consists of two parts, since the Yandex.Metrica cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier. +`{layer}-{shard}` is the shard identifier. In this example it consists of two parts, since the Yandex.Metrica cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier. -``hits`` - is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it doesn't change after a RENAME query. +`hits` is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it doesn't change after a RENAME query. The replica name identifies different replicas of the same table. You can use the server name for this, as in the example. The name only needs to be unique within each shard. @@ -111,10 +103,9 @@ Run CREATE TABLE on each replica. This query creates a new replicated table, or If you add a new replica after the table already contains some data on other replicas, the data will be copied from the other replicas to the new one after running the query. In other words, the new replica syncs itself with the others. -To delete a replica, run DROP TABLE. However, only one replica is deleted - the one that resides on the server where you run the query. +To delete a replica, run DROP TABLE. However, only one replica is deleted – the one that resides on the server where you run the query. -Recovery after failures -~~~~~~~~~~~~~~~~~~~~~~~ +## Recovery after failures If ZooKeeper is unavailable when a server starts, replicated tables switch to read-only mode. The system periodically attempts to connect to ZooKeeper. @@ -130,57 +121,55 @@ When the server starts (or establishes a new session with ZooKeeper), it only ch If the local set of data differs too much from the expected one, a safety mechanism is triggered. The server enters this in the log and refuses to launch. The reason for this is that this case may indicate a configuration error, such as if a replica on a shard was accidentally configured like a replica on a different shard. However, the thresholds for this mechanism are set fairly low, and this situation might occur during normal failure recovery. In this case, data is restored semi-automatically - by "pushing a button". -To start recovery, create the node ``/path_to_table/replica_name/flags/force_restore_data`` in ZooKeeper with any content or run command to recover all replicated tables: +To start recovery, create the node `/path_to_table/replica_name/flags/force_restore_data` in ZooKeeper with any content, or run the command to restore all replicated tables: -.. code-block:: text +```bash +sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data +``` - sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data +Then restart the server. On start, the server deletes these flags and starts recovery. -Then launch the server. On start, the server deletes these flags and starts recovery. - -Recovery after complete data loss -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Recovery after complete data loss If all data and metadata disappeared from one of the servers, follow these steps for recovery: -#. Install ClickHouse on the server. Define substitutions correctly in the config file that contains the shard identifier and replicas, if you use them. -#. If you had unreplicated tables that must be manually duplicated on the servers, copy their data from a replica (in the directory /var/lib/clickhouse/data/db_name/table_name/). -#. Copy table definitions located in /var/lib/clickhouse/metadata/. from a replica. If a shard or replica identifier is defined explicitly in the table definitions, correct it so that it corresponds to this replica. (Alternatively, launch the server and make all the ATTACH TABLE queries that should have been in the .sql files in /var/lib/clickhouse/metadata/.) -#. Create the ``/path_to_table/replica_name/flags/force_restore_data`` node in ZooKeeper with any content or run command to recover all replicated tables: ``sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data`` +1. Install ClickHouse on the server. Define substitutions correctly in the config file that contains the shard identifier and replicas, if you use them. +2. If you had unreplicated tables that must be manually duplicated on the servers, copy their data from a replica (in the directory /var/lib/clickhouse/data/db_name/table_name/). +3. Copy table definitions located in /var/lib/clickhouse/metadata/ from a replica. If a shard or replica identifier is defined explicitly in the table definitions, correct it so that it corresponds to this replica. (Alternatively, launch the server and make all the ATTACH TABLE queries that should have been in the .sql files in /var/lib/clickhouse/metadata/.) +4. To start recovery, create the ZooKeeper node /path_to_table/replica_name/flags/force_restore_data with any content, or run the command to restore all replicated tables: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` -Then launch the server (restart it if it is already running). Data will be downloaded from replicas. +Then start the server (restart, if it is already running). Data will be downloaded from replicas. -An alternative recovery option is to delete information about the lost replica from ZooKeeper ( ``/path_to_table/replica_name``), then create the replica again as described in "Creating replicated tables". +An alternative recovery option is to delete information about the lost replica from ZooKeeper ( `/path_to_table/replica_name`), then create the replica again as described in "Creating replicated tables". There is no restriction on network bandwidth during recovery. Keep this in mind if you are restoring many replicas at once. -Converting from MergeTree to ReplicatedMergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Converting from MergeTree to ReplicatedMergeTree -From here on, we use ``MergeTree`` to refer to all the table engines in the ``MergeTree`` family, including ``ReplicatedMergeTree``. +We use the term `MergeTree` to refer to all table engines in the ` MergeTree family`, the same as for ` ReplicatedMergeTree`. If you had a MergeTree table that was manually replicated, you can convert it to a replicatable table. You might need to do this if you have already collected a large amount of data in a MergeTree table and now you want to enable replication. If the data differs on various replicas, first sync it, or delete this data on all the replicas except one. Rename the existing MergeTree table, then create a ReplicatedMergeTree table with the old name. -Move the data from the old table to the 'detached' subdirectory inside the directory with the new table data (``/var/lib/clickhouse/data/db_name/table_name/``). +Move the data from the old table to the 'detached' subdirectory inside the directory with the new table data (`/var/lib/clickhouse/data/db_name/table_name/`). Then run ALTER TABLE ATTACH PARTITION on one of the replicas to add these data parts to the working set. -If exactly the same parts exist on the other replicas, they are added to the working set on them. If not, the parts are downloaded from the replica that has them. - -Converting from ReplicatedMergeTree to MergeTree -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Converting from ReplicatedMergeTree to MergeTree Create a MergeTree table with a different name. Move all the data from the directory with the ReplicatedMergeTree table data to the new table's data directory. Then delete the ReplicatedMergeTree table and restart the server. If you want to get rid of a ReplicatedMergeTree table without launching the server: -* Delete the corresponding .sql file in the metadata directory (``/var/lib/clickhouse/metadata/``). -* Delete the corresponding path in ZooKeeper (``/path_to_table/replica_name``). + +- Delete the corresponding .sql file in the metadata directory (`/var/lib/clickhouse/metadata/`). +- Delete the corresponding path in ZooKeeper (`/path_to_table/replica_name`). After this, you can launch the server, create a MergeTree table, move the data to its directory, and then restart the server. -Recovery when metadata in the ZooKeeper cluster is lost or damaged -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Recovery when metadata in the ZooKeeper cluster is lost or damaged + +If the data in ZooKeeper was lost or damaged, you can save data by moving it to an unreplicated table as described above. + +If exactly the same parts exist on the other replicas, they are added to the working set on them. If not, the parts are downloaded from the replica that has them. -If you lost ZooKeeper, you can save data by moving it to an unreplicated table as described above. diff --git a/docs/en/table_engines/set.rst b/docs/en/table_engines/set.md similarity index 99% rename from docs/en/table_engines/set.rst rename to docs/en/table_engines/set.md index 87ba8f18e83..d207fe0d79e 100644 --- a/docs/en/table_engines/set.rst +++ b/docs/en/table_engines/set.md @@ -1,5 +1,4 @@ -Set ---- +# Set A data set that is always in RAM. It is intended for use on the right side of the IN operator (see the section "IN operators"). @@ -9,3 +8,4 @@ But you can't perform SELECT from the table. The only way to retrieve data is by Data is always located in RAM. For INSERT, the blocks of inserted data are also written to the directory of tables on the disk. When starting the server, this data is loaded to RAM. In other words, after restarting, the data remains in place. For a rough server restart, the block of data on the disk might be lost or damaged. In the latter case, you may need to manually delete the file with damaged data. + diff --git a/docs/en/table_engines/summingmergetree.md b/docs/en/table_engines/summingmergetree.md new file mode 100644 index 00000000000..19f690ef9c1 --- /dev/null +++ b/docs/en/table_engines/summingmergetree.md @@ -0,0 +1,43 @@ +# SummingMergeTree + +This engine differs from `MergeTree` in that it totals data while merging. + +```sql +SummingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192) +``` + +The columns to total are implicit. When merging, all rows with the same primary key value (in the example, OrderId, EventDate, BannerID, ...) have their values totaled in numeric columns that are not part of the primary key. + +```sql +SummingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192, (Shows, Clicks, Cost, ...)) +``` + +The columns to total are set explicitly (the last parameter – Shows, Clicks, Cost, ...). When merging, all rows with the same primary key value have their values totaled in the specified columns. The specified columns also must be numeric and must not be part of the primary key. + +If the values were null in all of these columns, the row is deleted. (The exception is cases when the data part would not have any rows left in it.) + +For the other rows that are not part of the primary key, the first value that occurs is selected when merging. + +Summation is not performed for a read operation. If it is necessary, write the appropriate GROUP BY. + +In addition, a table can have nested data structures that are processed in a special way. +If the name of a nested table ends in 'Map' and it contains at least two columns that meet the following criteria: + +- The first table is numeric ((U)IntN, Date, DateTime), which we'll refer to as the 'key'. +- The other columns are arithmetic ((U)IntN, Float32/64), which we'll refer to as '(values...)'. Then this nested table is interpreted as a mapping of key =`>` (values...), and when merging its rows, the elements of two data sets are merged by 'key' with a summation of the corresponding (values...). + +Examples: + +```text +[(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] +[(1, 100)] + [(1, 150)] -> [(1, 250)] +[(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] +[(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] +``` + +For aggregation of Map, use the function sumMap(key, value). + +For nested data structures, you don't need to specify the columns as a list of columns for totaling. + +This table engine is not particularly useful. Remember that when saving just pre-aggregated data, you lose some of the system's advantages. + diff --git a/docs/en/table_engines/summingmergetree.rst b/docs/en/table_engines/summingmergetree.rst deleted file mode 100644 index 26ed69ae5a3..00000000000 --- a/docs/en/table_engines/summingmergetree.rst +++ /dev/null @@ -1,44 +0,0 @@ -SummingMergeTree ----------------- - -This engine differs from ``MergeTree`` in that it totals data while merging. - -.. code-block:: sql - - SummingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192) - -The columns to total are implicit. When merging, all rows with the same primary key value (in the example, OrderId, EventDate, BannerID, ...) have their values totaled in numeric columns that are not part of the primary key. - -.. code-block:: sql - - SummingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192, (Shows, Clicks, Cost, ...)) - -The columns to total are set explicitly (the last parameter - Shows, Clicks, Cost, ...). When merging, all rows with the same primary key value have their values totaled in the specified columns. The specified columns also must be numeric and must not be part of the primary key. - -If the values were null in all of these columns, the row is deleted. (The exception is cases when the data part would not have any rows left in it.) - -For the other rows that are not part of the primary key, the first value that occurs is selected when merging. - -Summation is not performed for a read operation. If it is necessary, write the appropriate GROUP BY. - -In addition, a table can have nested data structures that are processed in a special way. -If the name of a nested table ends in 'Map' and it contains at least two columns that meet the following criteria: -* for the first table, numeric ((U)IntN, Date, DateTime), we'll refer to it as 'key' -* for other tables, arithmetic ((U)IntN, Float32/64), we'll refer to it as '(values...)' - -Then this nested table is interpreted as a mapping of key => (values...), and when merging its rows, the elements of two data sets are merged by 'key' with a summation of the corresponding (values...). - -Examples: - -.. code-block:: text - - [(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] - [(1, 100)] + [(1, 150)] -> [(1, 250)] - [(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] - [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] - -For aggregating Map use function sumMap(key, value). - -For nested data structures, you don't need to specify the columns as a list of columns for totaling. - -This table engine is not particularly useful. Remember that when saving just pre-aggregated data, you lose some of the system's advantages. diff --git a/docs/en/table_engines/tinylog.rst b/docs/en/table_engines/tinylog.md similarity index 77% rename from docs/en/table_engines/tinylog.rst rename to docs/en/table_engines/tinylog.md index d4f10110f45..577596e2d96 100644 --- a/docs/en/table_engines/tinylog.rst +++ b/docs/en/table_engines/tinylog.md @@ -1,18 +1,19 @@ -TinyLog -------- +# TinyLog The simplest table engine, which stores data on a disk. Each column is stored in a separate compressed file. When writing, data is appended to the end of files. Concurrent data access is not restricted in any way: - - If you are simultaneously reading from a table and writing to it in a different query, the read operation will complete with an error. - - If you are writing to a table in multiple queries simultaneously, the data will be broken. + +- If you are simultaneously reading from a table and writing to it in a different query, the read operation will complete with an error. +- If you are writing to a table in multiple queries simultaneously, the data will be broken. The typical way to use this table is write-once: first just write the data one time, then read it as many times as needed. Queries are executed in a single stream. In other words, this engine is intended for relatively small tables (recommended up to 1,000,000 rows). It makes sense to use this table engine if you have many small tables, since it is simpler than the Log engine (fewer files need to be opened). The situation when you have a large number of small tables guarantees poor productivity, but may already be used when working with another DBMS, and you may find it easier to switch to using TinyLog types of tables. -Indexes are not supported. +**Indexes are not supported.** In Yandex.Metrica, TinyLog tables are used for intermediary data that is processed in small batches. + diff --git a/docs/en/table_engines/view.md b/docs/en/table_engines/view.md new file mode 100644 index 00000000000..91f0ce9fdb3 --- /dev/null +++ b/docs/en/table_engines/view.md @@ -0,0 +1,4 @@ +# View + +Used for implementing views (for more information, see the `CREATE VIEW query`). It does not store data, but only stores the specified `SELECT` query. When reading from a table, it runs this query (and deletes all unnecessary columns from the query). + diff --git a/docs/en/table_engines/view.rst b/docs/en/table_engines/view.rst deleted file mode 100644 index 08c06b85969..00000000000 --- a/docs/en/table_engines/view.rst +++ /dev/null @@ -1,4 +0,0 @@ -View ----- - -Used for implementing views (for more information, see the ``CREATE VIEW`` query). It does not store data, but only stores the specified ``SELECT`` query. When reading from a table, it runs this query (and deletes all unnecessary columns from the query). diff --git a/docs/en/table_functions/index.rst b/docs/en/table_functions/index.md similarity index 87% rename from docs/en/table_functions/index.rst rename to docs/en/table_functions/index.md index 196794a7204..6a0139f7866 100644 --- a/docs/en/table_functions/index.rst +++ b/docs/en/table_functions/index.md @@ -1,11 +1,13 @@ -Table functions -=============== +# Table functions Table functions can be specified in the FROM clause instead of the database and table names. Table functions can only be used if 'readonly' is not set. Table functions aren't related to other functions. +```eval_rst .. toctree:: :glob: * +``` + diff --git a/docs/en/table_functions/merge.md b/docs/en/table_functions/merge.md new file mode 100644 index 00000000000..d1ddc12dc97 --- /dev/null +++ b/docs/en/table_functions/merge.md @@ -0,0 +1,6 @@ +# merge + +`merge(db_name, 'tables_regexp')` – Creates a temporary Merge table. For more information, see the section "Table engines, Merge". + +The table structure is taken from the first table encountered that matches the regular expression. + diff --git a/docs/en/table_functions/merge.rst b/docs/en/table_functions/merge.rst deleted file mode 100644 index f4e1246a363..00000000000 --- a/docs/en/table_functions/merge.rst +++ /dev/null @@ -1,6 +0,0 @@ -merge ------ - -``merge(db_name, 'tables_regexp')`` creates a temporary Merge table. For more information, see the section "Table engines, Merge". - -The table structure is taken from the first table encountered that matches the regular expression. diff --git a/docs/en/table_functions/remote.md b/docs/en/table_functions/remote.md new file mode 100644 index 00000000000..99b0c7bb116 --- /dev/null +++ b/docs/en/table_functions/remote.md @@ -0,0 +1,78 @@ + + +# remote + +Allows you to access remote servers without creating a `Distributed` table. + +Signatures: + +```sql +remote('addresses_expr', db, table[, 'user'[, 'password']]) +remote('addresses_expr', db.table[, 'user'[, 'password']]) +``` + +`addresses_expr` – An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`. The host can be specified as the server name, or as the IPv4 or IPv6 address. An IPv6 address is specified in square brackets. The port is the TCP port on the remote server. If the port is omitted, it uses `tcp_port` from the server's config file (by default, 9000). + +
+ +The port is required for an IPv6 address. + +
+ +Examples: + +```text +example01-01-1 +example01-01-1:9000 +localhost +127.0.0.1 +[::]:9000 +[2a02:6b8:0:1111::11]:9000 +``` + +Multiple addresses can be comma-separated. In this case, ClickHouse will use distributed processing, so it will send the query to all specified addresses (like to shards with different data). + +Example: + +```text +example01-01-1,example01-02-1 +``` + +Part of the expression can be specified in curly brackets. The previous example can be written as follows: + +```text +example01-0{1,2}-1 +``` + +Curly brackets can contain a range of numbers separated by two dots (non-negative integers). In this case, the range is expanded to a set of values that generate shard addresses. If the first number starts with zero, the values are formed with the same zero alignment. The previous example can be written as follows: + +```text +example01-{01..02}-1 +``` + +If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets. + +Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting. + +Example: + +```text +example01-{01..02}-{1|2} +``` + +This example specifies two shards that each have two replicas. + +The number of addresses generated is limited by a constant. Right now this is 1000 addresses. + +Using the `remote` table function is less optimal than creating a `Distributed` table, because in this case, the server connection is re-established for every request. In addition, if host names are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and don't use the `remote` table function. + +The `remote` table function can be useful in the following cases: + +- Accessing a specific server for data comparison, debugging, and testing. +- Queries between various ClickHouse clusters for research purposes. +- Infrequent distributed requests that are made manually. +- Distributed requests where the set of servers is re-defined each time. + +If the user is not specified, `default` is used. +If the password is not specified, an empty password is used. + diff --git a/docs/en/table_functions/remote.rst b/docs/en/table_functions/remote.rst deleted file mode 100644 index 7b2a58f26ca..00000000000 --- a/docs/en/table_functions/remote.rst +++ /dev/null @@ -1,73 +0,0 @@ -remote ------- - -``remote('addresses_expr', db, table[, 'user'[, 'password']])`` - -or - -``remote('addresses_expr', db.table[, 'user'[, 'password']])`` - -- Allows accessing a remote server without creating a Distributed table. - -``addresses_expr`` - An expression that generates addresses of remote servers. - -This may be just one server address. The server address is host:port, or just the host. The host can be specified as the server name, or as the IPv4 or IPv6 address. An IPv6 address is specified in square brackets. The port is the TCP port on the remote server. If the port is omitted, it uses tcp_port from the server's config file (by default, 9000). - -Note: As an exception, when specifying an IPv6 address, the port is required. - -Examples: - -.. code-block:: text - - example01-01-1 - example01-01-1:9000 - localhost - 127.0.0.1 - [::]:9000 - [2a02:6b8:0:1111::11]:9000 - -Multiple addresses can be comma-separated. In this case, the query goes to all the specified addresses (like to shards with different data) and uses distributed processing. - -Example: - -.. code-block:: text - - example01-01-1,example01-02-1 - -Part of the expression can be specified in curly brackets. The previous example can be written as follows: - -.. code-block:: text - - example01-0{1,2}-1 - -Curly brackets can contain a range of numbers separated by two dots (non-negative integers). In this case, the range is expanded to a set of values that generate shard addresses. If the first number starts with zero, the values are formed with the same zero alignment. -The previous example can be written as follows: - -.. code-block:: text - - example01-{01..02}-1 - -If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets. - -Addresses and fragments in curly brackets can be separated by the pipe (|) symbol. In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the 'load_balancing' setting. - -Example: - -.. code-block:: text - - example01-{01..02}-{1|2} - -This example specifies two shards that each have two replicas. - -The number of addresses generated is limited by a constant. Right now this is 1000 addresses. - -Using the 'remote' table function is less optimal than creating a Distributed table, because in this case, the server connection is re-established for every request. In addition, if host names are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the Distributed table ahead of time, and don't use the 'remote' table function. - -The 'remote' table function can be useful in the following cases: - * Accessing a specific server for data comparison, debugging, and testing. - * Queries between various ClickHouse clusters for research purposes. - * Infrequent distributed requests that are made manually. - * Distributed requests where the set of servers is re-defined each time. - -The username can be omitted. In this case, the 'default' username is used. -The password can be omitted. In this case, an empty password is used. diff --git a/docs/ru/table_engines/dictionary.md b/docs/ru/table_engines/dictionary.md index 1918c611344..40ef01beb01 100644 --- a/docs/ru/table_engines/dictionary.md +++ b/docs/ru/table_engines/dictionary.md @@ -38,8 +38,9 @@ ``` Проверяем: + ```sql -:) select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products'; +select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products'; SELECT name, @@ -52,7 +53,8 @@ SELECT source FROM system.dictionaries WHERE name = 'products' - +``` +``` ┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐ │ products │ Flat │ UInt64 │ ['title'] │ ['String'] │ 23065376 │ 175032 │ ODBC: .products │ └──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ @@ -66,9 +68,9 @@ WHERE name = 'products' Синтаксис: `CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` Попробуем: -```sql -:) create table products (product_id UInt64, title String) Engine = Dictionary(products); +```sql +create table products (product_id UInt64, title String) Engine = Dictionary(products); CREATE TABLE products ( @@ -76,20 +78,23 @@ CREATE TABLE products title String, ) ENGINE = Dictionary(products) - +``` +``` Ok. 0 rows in set. Elapsed: 0.004 sec. ``` Проверим что у нас в таблице? + ```sql -:) select * from products limit 1; +select * from products limit 1; SELECT * FROM products LIMIT 1 - +``` +``` ┌────product_id─┬─title───────────┐ │ 152689 │ Некоторый товар │ └───────────────┴─────────────────┘