From ca1484ae95dffa7a5eab91f232e266f3e13c8a5d Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Wed, 13 Feb 2019 22:26:24 +0300 Subject: [PATCH 01/17] set exactly one arg --- .../MergeTree/MergeTreeSetSkippingIndex.cpp | 25 ++++++++----------- .../queries/0_stateless/00838_unique_index.sh | 6 ++--- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index bed74d0d640..3c3a414d000 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -31,12 +31,12 @@ void MergeTreeSetIndexGranule::serializeBinary(WriteBuffer & ostr) const { if (empty()) throw Exception( - "Attempt to write empty unique index `" + index.name + "`", ErrorCodes::LOGICAL_ERROR); + "Attempt to write empty set index `" + index.name + "`", ErrorCodes::LOGICAL_ERROR); const auto & columns = set->getSetElements(); const auto & size_type = DataTypePtr(std::make_shared()); - if (index.max_rows && size() > index.max_rows) + if (size() > index.max_rows) { size_type->serializeBinary(0, ostr); return; @@ -87,7 +87,7 @@ void MergeTreeSetIndexGranule::update(const Block & new_block, size_t * pos, UIn size_t rows_read = std::min(limit, new_block.rows() - *pos); - if (index.max_rows && size() > index.max_rows) + if (size() > index.max_rows) { *pos += rows_read; return; @@ -112,7 +112,7 @@ void MergeTreeSetIndexGranule::update(const Block & new_block, size_t * pos, UIn Block MergeTreeSetIndexGranule::getElementsBlock() const { - if (index.max_rows && size() > index.max_rows) + if (size() > index.max_rows) return index.header; return index.header.cloneWithColumns(set->getSetElements()); } @@ -169,12 +169,12 @@ bool SetIndexCondition::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx_granule) auto granule = std::dynamic_pointer_cast(idx_granule); if (!granule) throw Exception( - "Unique index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); + "Set index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); if (useless) return true; - if (index.max_rows && granule->size() > index.max_rows) + if (granule->size() > index.max_rows) return true; Block result = granule->getElementsBlock(); @@ -363,14 +363,11 @@ std::unique_ptr setIndexCreator( throw Exception("Index must have unique name", ErrorCodes::INCORRECT_QUERY); size_t max_rows = 0; - if (node->type->arguments) - { - if (node->type->arguments->children.size() > 1) - throw Exception("Unique index cannot have only 0 or 1 argument", ErrorCodes::INCORRECT_QUERY); - else if (node->type->arguments->children.size() == 1) - max_rows = typeid_cast( - *node->type->arguments->children[0]).value.get(); - } + if (!node->type->arguments || node->type->arguments->children.size() != 1) + throw Exception("Set index must have exactly one argument.", ErrorCodes::INCORRECT_QUERY); + else if (node->type->arguments->children.size() == 1) + max_rows = typeid_cast( + *node->type->arguments->children[0]).value.get(); ASTPtr expr_list = MergeTreeData::extractKeyExpressionList(node->expr->clone()); diff --git a/dbms/tests/queries/0_stateless/00838_unique_index.sh b/dbms/tests/queries/0_stateless/00838_unique_index.sh index dd4440bd5ce..f6bea4f083a 100755 --- a/dbms/tests/queries/0_stateless/00838_unique_index.sh +++ b/dbms/tests/queries/0_stateless/00838_unique_index.sh @@ -16,9 +16,9 @@ CREATE TABLE test.set_idx s String, e Enum8('a' = 1, 'b' = 2, 'c' = 3), dt Date, - INDEX idx_all (i32, i32 + f64, d, s, e, dt) TYPE set GRANULARITY 1, - INDEX idx_all2 (i32, i32 + f64, d, s, e, dt) TYPE set GRANULARITY 2, - INDEX idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE set GRANULARITY 3 + INDEX idx_all (i32, i32 + f64, d, s, e, dt) TYPE set(2) GRANULARITY 1, + INDEX idx_all2 (i32, i32 + f64, d, s, e, dt) TYPE set(4) GRANULARITY 2, + INDEX idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE set(6) GRANULARITY 3 ) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 2;" From 04a62f3df4b00e978be45d6425af1db1d03191e0 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Wed, 13 Feb 2019 22:29:31 +0300 Subject: [PATCH 02/17] set args --- docs/en/operations/table_engines/mergetree.md | 6 ++---- docs/ru/operations/table_engines/mergetree.md | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 948d63ff7d8..7d86776d27b 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -252,7 +252,7 @@ CREATE TABLE table_name s String, ... INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, - INDEX b (u64 * length(s)) TYPE set GRANULARITY 4 + INDEX b (u64 * length(s)) TYPE set(1000) GRANULARITY 4 ) ENGINE = MergeTree() ... ``` @@ -269,12 +269,10 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of the data like the primary key. * `set(max_rows)` -Stores unique values of the specified expression (no more than `max_rows` rows), use them to check if the `WHERE` expression is not satisfiable on a block of the data. -If `max_rows=0`, then there are no limits for storing values. `set` without parameters is equal to `set(0)`. +Stores unique values of the specified expression (no more than `max_rows` rows), use them to check if the `WHERE` expression is not satisfiable on a block of the data. ```sql INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 -INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set GRANULARITY 4 INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 ``` diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index 3c4f84d1c8c..258c6fc7ce1 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -243,7 +243,7 @@ CREATE TABLE table_name s String, ... INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, - INDEX b (u64 * length(s), i32) TYPE set GRANULARITY 4 + INDEX b (u64 * length(s), i32) TYPE set(1000) GRANULARITY 4 ) ENGINE = MergeTree() ... ``` @@ -261,7 +261,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 * `set(max_rows)` Хранит уникальные значения выражения на блоке в количестве не более `max_rows`, используя их для пропуска блоков, оценивая выполнимость `WHERE` выражения на хранимых данных. -Если `max_rows=0`, то хранит значения выражения без ограничений. Если параметров не передано, то полагается `max_rows=0`. + Примеры From 5f40ae53e37e961858a86cb2139ab3b8579473ff Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Wed, 13 Feb 2019 22:33:10 +0300 Subject: [PATCH 03/17] fix --- docs/ru/operations/table_engines/mergetree.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index 258c6fc7ce1..aca22f61563 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -263,7 +263,6 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Хранит уникальные значения выражения на блоке в количестве не более `max_rows`, используя их для пропуска блоков, оценивая выполнимость `WHERE` выражения на хранимых данных. - Примеры ```sql INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE minmax GRANULARITY 4 From 55df6f0a1a7b5fcf02b7ef998ee07dda1b1575c3 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Wed, 13 Feb 2019 22:33:58 +0300 Subject: [PATCH 04/17] fix docs --- docs/ru/operations/table_engines/mergetree.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index aca22f61563..22df2d45e64 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -266,7 +266,6 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Примеры ```sql INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE minmax GRANULARITY 4 -INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set GRANULARITY 4 INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 ``` From f8c0f4697c11499ff791daf359cb770b1ec12502 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Thu, 14 Feb 2019 11:49:31 +0300 Subject: [PATCH 05/17] create test --- .../00907_set_index_max_rows.reference | 1 + .../0_stateless/00907_set_index_max_rows.sh | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference create mode 100755 dbms/tests/queries/0_stateless/00907_set_index_max_rows.sh diff --git a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference new file mode 100644 index 00000000000..a80322c42a2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference @@ -0,0 +1 @@ + "rows_read": 0, diff --git a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.sh b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.sh new file mode 100755 index 00000000000..ed2a732c74f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.set_idx;" + +$CLICKHOUSE_CLIENT -n --query=" +SET allow_experimental_data_skipping_indices = 1; +CREATE TABLE test.set_idx +( + u64 UInt64, + i32 Int32, + INDEX idx (i32) TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY u64 +SETTINGS index_granularity = 6;" + +$CLICKHOUSE_CLIENT --query=" +INSERT INTO test.set_idx +SELECT number, number FROM system.numbers LIMIT 100" + +# simple select +$CLICKHOUSE_CLIENT --query="SELECT * FROM test.set_idx WHERE i32 > 0 FORMAT JSON" | grep "rows_read" + + +$CLICKHOUSE_CLIENT --query="DROP TABLE test.set_idx;" \ No newline at end of file From 60158d06ede348ca440863da46bb5c6ff64a2a71 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Thu, 14 Feb 2019 11:50:17 +0300 Subject: [PATCH 06/17] fix test --- .../queries/0_stateless/00907_set_index_max_rows.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference index a80322c42a2..4f09265b3b4 100644 --- a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference +++ b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference @@ -1 +1 @@ - "rows_read": 0, + "rows_read": 100, From 683314b69b668586f3993b1d48b980f3a2129bcb Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Thu, 14 Feb 2019 12:06:32 +0300 Subject: [PATCH 07/17] fix set --- dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index 3c3a414d000..89d2b38e550 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -171,10 +171,7 @@ bool SetIndexCondition::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx_granule) throw Exception( "Set index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); - if (useless) - return true; - - if (granule->size() > index.max_rows) + if (useless || !granule->size() || granule->size() > index.max_rows) return true; Block result = granule->getElementsBlock(); From 64e0732b4abf7dddb22b22fc2451c935534683e6 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Thu, 14 Feb 2019 12:13:04 +0300 Subject: [PATCH 08/17] fixed --- dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp | 7 ++----- .../queries/0_stateless/00907_set_index_max_rows.reference | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp index f2bcdb4a1ff..2b60901f8df 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp @@ -39,13 +39,11 @@ void MergeTreeMinMaxGranule::serializeBinary(WriteBuffer & ostr) const void MergeTreeMinMaxGranule::deserializeBinary(ReadBuffer & istr) { parallelogram.clear(); + Field min_val, max_val; for (size_t i = 0; i < index.columns.size(); ++i) { const DataTypePtr & type = index.data_types[i]; - - Field min_val; type->deserializeBinary(min_val, istr); - Field max_val; type->deserializeBinary(max_val, istr); parallelogram.emplace_back(min_val, true, max_val, true); @@ -61,11 +59,10 @@ void MergeTreeMinMaxGranule::update(const Block & block, size_t * pos, UInt64 li size_t rows_read = std::min(limit, block.rows() - *pos); + Field field_min, field_max; for (size_t i = 0; i < index.columns.size(); ++i) { const auto & column = block.getByName(index.columns[i]).column; - - Field field_min, field_max; column->cut(*pos, rows_read)->getExtremes(field_min, field_max); if (parallelogram.size() <= i) diff --git a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference index 4f09265b3b4..3ee41d6fea1 100644 --- a/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference +++ b/dbms/tests/queries/0_stateless/00907_set_index_max_rows.reference @@ -1 +1 @@ - "rows_read": 100, + "rows_read": 100, From edff16aa15aab18e17a02870eea5aa8b49d7280a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 14 Feb 2019 15:12:28 +0300 Subject: [PATCH 09/17] Minor docs changes (#4366) --- docs/en/development/contrib.md | 33 ++++++++++++++++++++++ docs/en/operations/index.md | 2 +- docs/fa/development/contrib.md | 1 + docs/ru/development/contrib.md | 33 ++++++++++++++++++++++ docs/ru/introduction/info.md | 9 ++++++ docs/ru/operations/index.md | 2 +- docs/toc_en.yml | 1 + docs/toc_fa.yml | 1 + docs/toc_ru.yml | 2 ++ docs/toc_zh.yml | 1 + docs/tools/mkdocs-material-theme/base.html | 11 +++++++- docs/zh/development/contrib.md | 1 + 12 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 docs/en/development/contrib.md create mode 120000 docs/fa/development/contrib.md create mode 100644 docs/ru/development/contrib.md create mode 100644 docs/ru/introduction/info.md create mode 120000 docs/zh/development/contrib.md diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md new file mode 100644 index 00000000000..cfbdd28dbf3 --- /dev/null +++ b/docs/en/development/contrib.md @@ -0,0 +1,33 @@ +# Third-Party Libraries Used + +| Library | License | +| ------- | ------- | +| base64 | [BSD 2-Clause License](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | +| boost | [Boost Software License 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | +| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | +| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | +| cctz | [Apache License 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | +| double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | +| FastMemcpy | [MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | +| googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) | +| libbtrie | [BSD 2-Clause License](https://github.com/yandex/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | +| libcxxabi | [BSD + MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | +| libdivide | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | +| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) +| libhdfs3 | [Apache License 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | +| libmetrohash | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | +| libpcg-random | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | +| libressl | [OpenSSL License](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | +| librdkafka | [BSD 2-Clause License](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | +| libwidechar\_width | [CC0 1.0 Universal](https://github.com/yandex/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | +| llvm | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | +| lz4 | [BSD 2-Clause License](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | +| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | +| murmurhash | [Public Domain](https://github.com/yandex/ClickHouse/blob/master/contrib/murmurhash/LICENSE) +| pdqsort | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/pdqsort/license.txt) | +| poco | [Boost Software License - Version 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | +| protobuf | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | +| re2 | [BSD 3-Clause License](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | +| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | +| zlib-ng | [Zlib License](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | +| zstd | [BSD 3-Clause License](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md index b864f327a2e..547fc4de260 100644 --- a/docs/en/operations/index.md +++ b/docs/en/operations/index.md @@ -6,7 +6,7 @@ ClickHouse operations manual consists of the following major sections: - [Monitoring](monitoring.md) - [Troubleshooting](troubleshooting.md) - [Usage Recommendations](tips.md) - - [ClickHouse Update](update.md) + - [Update Procedure](update.md) - [Access Rights](access_rights.md) - [Data Backup](backup.md) - [Configuration Files](configuration_files.md) diff --git a/docs/fa/development/contrib.md b/docs/fa/development/contrib.md new file mode 120000 index 00000000000..4749f95f9ef --- /dev/null +++ b/docs/fa/development/contrib.md @@ -0,0 +1 @@ +../../en/development/contrib.md \ No newline at end of file diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md new file mode 100644 index 00000000000..f7afd8bb0a8 --- /dev/null +++ b/docs/ru/development/contrib.md @@ -0,0 +1,33 @@ +# Используемые сторонние библиотеки + +| Библиотека | Лицензия | +| ------- | ------- | +| base64 | [BSD 2-Clause License](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | +| boost | [Boost Software License 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | +| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | +| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | +| cctz | [Apache License 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | +| double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | +| FastMemcpy | [MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | +| googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) | +| libbtrie | [BSD 2-Clause License](https://github.com/yandex/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | +| libcxxabi | [BSD + MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | +| libdivide | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | +| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) +| libhdfs3 | [Apache License 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | +| libmetrohash | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | +| libpcg-random | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | +| libressl | [OpenSSL License](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | +| librdkafka | [BSD 2-Clause License](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | +| libwidechar\_width | [CC0 1.0 Universal](https://github.com/yandex/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | +| llvm | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | +| lz4 | [BSD 2-Clause License](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | +| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | +| murmurhash | [Public Domain](https://github.com/yandex/ClickHouse/blob/master/contrib/murmurhash/LICENSE) +| pdqsort | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/pdqsort/license.txt) | +| poco | [Boost Software License - Version 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | +| protobuf | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | +| re2 | [BSD 3-Clause License](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | +| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | +| zlib-ng | [Zlib License](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | +| zstd | [BSD 3-Clause License](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/ru/introduction/info.md b/docs/ru/introduction/info.md new file mode 100644 index 00000000000..69b40c1776d --- /dev/null +++ b/docs/ru/introduction/info.md @@ -0,0 +1,9 @@ +# Информационная поддержка +Информационная поддержка ClickHouse осуществляется на всей территории Российской Федерации без ограничений посредством использования телефонной связи и средств электронной почты на русском языке в круглосуточном режиме: + +* Адрес электронной почты: +* Телефон: 8-800-250-96-39 (звонки бесплатны из всех регионов России) + +

















+ +[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/info/) diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 9ad2fc5ff76..a10f7c377b1 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -6,7 +6,7 @@ - [Мониторинг](monitoring.md) - [Решение проблем](troubleshooting.md) - [Советы по эксплуатации](tips.md) - - [Обновление ClickHouse](update.md) + - [Процедура обновления](update.md) - [Права доступа](access_rights.md) - [Резервное копирование](backup.md) - [Конфигурационные файлы](configuration_files.md) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index d27929eb855..aa9a7e2060f 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -192,6 +192,7 @@ nav: - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Write C++ code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' + - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': - 'Roadmap': 'roadmap.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index f51c11125c8..0b84040282a 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -192,6 +192,7 @@ nav: - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Write C++ code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' + - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': - 'Roadmap': 'roadmap.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index c3a7afb8e05..2840eaed6db 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -6,6 +6,7 @@ nav: - 'Особенности ClickHouse, которые могут считаться недостатками': 'introduction/features_considered_disadvantages.md' - 'Производительность': 'introduction/performance.md' - 'Постановка задачи в Яндекс.Метрике': 'introduction/ya_metrika_task.md' + - 'Информационная поддержка': 'introduction/info.md' - 'Начало работы': - 'Установка и запуск': 'getting_started/index.md' @@ -192,6 +193,7 @@ nav: - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - 'Как писать код на C++': 'development/style.md' - 'Как запустить тесты': 'development/tests.md' + - 'Сторонние библиотеки': 'development/contrib.md' - 'Что нового': - 'Changelog': 'changelog.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 30fc4a2fa66..64cb2446f41 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -191,6 +191,7 @@ nav: - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' - '如何编写C++代码': 'development/style.md' - '如何运行ClickHouse测试': 'development/tests.md' + - '使用的第三方库': 'development/contrib.md' - '新功能特性': - '路线图': 'roadmap.md' diff --git a/docs/tools/mkdocs-material-theme/base.html b/docs/tools/mkdocs-material-theme/base.html index 05708186299..91a10d8e6c3 100644 --- a/docs/tools/mkdocs-material-theme/base.html +++ b/docs/tools/mkdocs-material-theme/base.html @@ -223,6 +223,7 @@ }); } ready(function () { + var i; {% if config.extra.single_page and page.content %} document.getElementById("content").innerHTML = {{ page.content|tojson|safe }}; document.getElementsByClassName('md-footer')[0].style.display = 'block'; @@ -235,7 +236,7 @@ }); var beforePrint = function() { var details = document.getElementsByTagName("details"); - for (var i = 0; i < details.length; ++i) { + for (i = 0; i < details.length; ++i) { details[i].open = 1; } }; @@ -247,6 +248,14 @@ }); } window.onbeforeprint = beforePrint; + + var name = window.location.hostname.split('.')[0]; + var feedback_address = name + '-feedback' + '@yandex-team.ru'; + var feedback_email = document.getElementsByClassName('feedback-email'); + for (i = 0; i < feedback_email.length; i++) { + feedback_email[i].setAttribute('href', 'mailto:' + feedback_address); + feedback_email[i].innerHTML = feedback_address; + } }); diff --git a/docs/zh/development/contrib.md b/docs/zh/development/contrib.md new file mode 120000 index 00000000000..4749f95f9ef --- /dev/null +++ b/docs/zh/development/contrib.md @@ -0,0 +1 @@ +../../en/development/contrib.md \ No newline at end of file From b7a4456007ec883e5bedbc591b18f2e8423be741 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 Feb 2019 16:05:44 +0300 Subject: [PATCH 10/17] Avoid "Memory limit exceeded" during ATTACH TABLE query --- dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 100639a999c..186f254acfb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -460,6 +460,11 @@ void MergeTreeDataPart::makeCloneInDetached(const String & prefix) const void MergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) { + /// Memory should not be limited during ATTACH TABLE query. + /// This is already true at the server startup but must be also ensured for manual table ATTACH. + /// Motivation: memory for index is shared between queries - not belong to the query itself. + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + loadColumns(require_columns_checksums); loadChecksums(require_columns_checksums); loadIndex(); From 048f966cde96d22d828416b10a4ef4db25aacaf8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 Feb 2019 16:42:26 +0300 Subject: [PATCH 11/17] Added a test #4396 --- .../0_stateless/00899_attach_memory_limit.reference | 2 ++ .../queries/0_stateless/00899_attach_memory_limit.sql | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00899_attach_memory_limit.reference create mode 100644 dbms/tests/queries/0_stateless/00899_attach_memory_limit.sql diff --git a/dbms/tests/queries/0_stateless/00899_attach_memory_limit.reference b/dbms/tests/queries/0_stateless/00899_attach_memory_limit.reference new file mode 100644 index 00000000000..4e65c4e0d23 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00899_attach_memory_limit.reference @@ -0,0 +1,2 @@ +10000000 +10000000 diff --git a/dbms/tests/queries/0_stateless/00899_attach_memory_limit.sql b/dbms/tests/queries/0_stateless/00899_attach_memory_limit.sql new file mode 100644 index 00000000000..38bde7d9bdf --- /dev/null +++ b/dbms/tests/queries/0_stateless/00899_attach_memory_limit.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test.index_memory; +CREATE TABLE test.index_memory (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1; +INSERT INTO test.index_memory SELECT * FROM system.numbers LIMIT 10000000; +SELECT count() FROM test.index_memory; +DETACH TABLE test.index_memory; +SET max_memory_usage = 79000000; +ATTACH TABLE test.index_memory; +SELECT count() FROM test.index_memory; +DROP TABLE test.index_memory; From 2efb4bdca253d77183f9cd03b01a39d65eb26ae8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 Feb 2019 16:47:00 +0300 Subject: [PATCH 12/17] Slightly raised up the limit on max string and array size received from ZooKeeper --- dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp index ac049bcb8e5..4abb97f1ccb 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -14,6 +14,11 @@ #include +/// ZooKeeper has 1 MB node size and serialization limit by default, +/// but it can be raised up, so we have a slightly larger limit on our side. +#define MAX_STRING_OR_ARRAY_SIZE (1 << 28) /// 256 MiB + + namespace ProfileEvents { extern const Event ZooKeeperInit; @@ -322,7 +327,6 @@ void read(bool & x, ReadBuffer & in) void read(String & s, ReadBuffer & in) { - static constexpr int32_t max_string_size = 1 << 20; int32_t size = 0; read(size, in); @@ -336,7 +340,7 @@ void read(String & s, ReadBuffer & in) if (size < 0) throw Exception("Negative size while reading string from ZooKeeper", ZMARSHALLINGERROR); - if (size > max_string_size) + if (size > MAX_STRING_OR_ARRAY_SIZE) throw Exception("Too large string size while reading from ZooKeeper", ZMARSHALLINGERROR); s.resize(size); @@ -369,12 +373,11 @@ void read(Stat & stat, ReadBuffer & in) template void read(std::vector & arr, ReadBuffer & in) { - static constexpr int32_t max_array_size = 1 << 20; int32_t size = 0; read(size, in); if (size < 0) throw Exception("Negative size while reading array from ZooKeeper", ZMARSHALLINGERROR); - if (size > max_array_size) + if (size > MAX_STRING_OR_ARRAY_SIZE) throw Exception("Too large array size while reading from ZooKeeper", ZMARSHALLINGERROR); arr.resize(size); for (auto & elem : arr) From 21247ebfac582fd7e971f249e0ed3b90e43b333c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 14 Feb 2019 17:04:28 +0300 Subject: [PATCH 13/17] Better replica repair logic: clearing obsolete queue --- dbms/src/Common/ZooKeeper/ZooKeeper.cpp | 16 ++++++++++++++++ dbms/src/Common/ZooKeeper/ZooKeeper.h | 3 +++ dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 +++ 3 files changed, 22 insertions(+) diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp index 522667b8055..b624cc7fce5 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp @@ -523,6 +523,22 @@ int32_t ZooKeeper::tryMulti(const Coordination::Requests & requests, Coordinatio } +void ZooKeeper::removeChildren(const std::string & path) +{ + Strings children = getChildren(path); + while (!children.empty()) + { + Coordination::Requests ops; + for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) + { + ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + children.pop_back(); + } + multi(ops); + } +} + + void ZooKeeper::removeChildrenRecursive(const std::string & path) { Strings children = getChildren(path); diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h index 99d87dce694..48e77741e68 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.h +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h @@ -177,6 +177,9 @@ public: /// result would be the same as for the single call. void tryRemoveRecursive(const std::string & path); + /// Remove all children nodes (non recursive). + void removeChildren(const std::string & path); + /// Wait for the node to disappear or return immediately if it doesn't exist. void waitForDisappear(const std::string & path); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index fd851bba27c..b7742c645ed 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2036,6 +2036,9 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke if (source_replica.empty()) throw Exception("All replicas are lost", ErrorCodes::ALL_REPLICAS_LOST); + /// Clear obsolete queue that we no longer need. + zookeeper->removeChildren(replica_path + "/queue"); + /// Will do repair from the selected replica. cloneReplica(source_replica, source_is_lost_stat, zookeeper); /// If repair fails to whatever reason, the exception is thrown, is_lost will remain "1" and the replica will be repaired later. From 533665c9f2e9086c32cf6bc2fcf1338382d35945 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 14 Feb 2019 17:10:21 +0300 Subject: [PATCH 14/17] DOCAPI-4552: EN review of VersionedCollapsingMergeTree topic (#4395) --- .../versionedcollapsingmergetree.md | 80 +++++++++---------- .../versionedcollapsingmergetree.md | 1 - 2 files changed, 40 insertions(+), 41 deletions(-) delete mode 120000 docs/ru/operations/table_engines/versionedcollapsingmergetree.md diff --git a/docs/en/operations/table_engines/versionedcollapsingmergetree.md b/docs/en/operations/table_engines/versionedcollapsingmergetree.md index 90e90c96a4f..3a489632f36 100644 --- a/docs/en/operations/table_engines/versionedcollapsingmergetree.md +++ b/docs/en/operations/table_engines/versionedcollapsingmergetree.md @@ -3,12 +3,12 @@ This engine: -- Allows quick writing of continually changing states of objects. -- Deletes old states of objects in the background. It causes to significant reduction of the volume of storage. +- Allows quick writing of object states that are continually changing. +- Deletes old object states in the background. This significantly reduces the volume of storage. See the section [Collapsing](#collapsing) for details. -The engine inherits from [MergeTree](mergetree.md#table_engines-mergetree) and adds the logic of rows collapsing to data parts merge algorithm. `VersionedCollapsingMergeTree` solves the same problem as the [CollapsingMergeTree](collapsingmergetree.md) but uses another algorithm of collapsing. It allows inserting the data in any order with multiple threads. The particular `Version` column helps to collapse the rows properly even if they are inserted in the wrong order. `CollapsingMergeTree` allows only strictly consecutive insertion. +The engine inherits from [MergeTree](mergetree.md#table_engines-mergetree) and adds the logic for collapsing rows to the algorithm for merging data parts. `VersionedCollapsingMergeTree` serves the same purpose as [CollapsingMergeTree](collapsingmergetree.md) but uses a different collapsing algorithm that allows inserting the data in any order with multiple threads. In particular, the `Version` column helps to collapse the rows properly even if they are inserted in the wrong order. In contrast, `CollapsingMergeTree` allows only strictly consecutive insertion. ## Creating a Table @@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -For a description of query parameters, see [query description](../../query_language/create.md). +For a description of query parameters, see the [query description](../../query_language/create.md). **Engine Parameters** @@ -35,20 +35,20 @@ VersionedCollapsingMergeTree(sign, version) - `sign` — Name of the column with the type of row: `1` is a "state" row, `-1` is a "cancel" row. - Column data type should be `Int8`. + The column data type should be `Int8`. -- `version` — Name of the column with the version of object state. +- `version` — Name of the column with the version of the object state. - Column data type should be `UInt*`. + The column data type should be `UInt*`. **Query Clauses** -When creating a `VersionedCollapsingMergeTree` table, the same [clauses](mergetree.md) are required, as when creating a `MergeTree` table. +When creating a `VersionedCollapsingMergeTree` table, the same [clauses](mergetree.md) are required as when creating a `MergeTree` table.
Deprecated Method for Creating a Table !!! attention - Do not use this method in new projects and, if possible, switch the old projects to the method described above. + Do not use this method in new projects. If possible, switch the old projects to the method described above. ```sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] @@ -59,15 +59,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] VersionedCollapsingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, sign, version) ``` -All of the parameters excepting `sign` and `version` have the same meaning as in `MergeTree`. +All of the parameters except `sign` and `version` have the same meaning as in `MergeTree`. -- `sign` — Name of the column with the type of row: `1` — "state" row, `-1` — "cancel" row. +- `sign` — Name of the column with the type of row: `1` is a "state" row, `-1` is a "cancel" row. Column Data Type — `Int8`. -- `version` — Name of the column with the version of object state. +- `version` — Name of the column with the version of the object state. - Column data type should be `UInt*`. + The column data type should be `UInt*`.
@@ -75,11 +75,11 @@ All of the parameters excepting `sign` and `version` have the same meaning as in ### Data -Consider the situation where you need to save continually changing data for some object, it is reasonable to have one row for an object and update it at any change. Update operation is expensive and slow for DBMS because it requires rewriting of the data in the storage. If you need to write data quickly, update not acceptable, but you can write the changes of an object sequentially as follows. +Consider a situation where you need to save continually changing data for some object. It is reasonable to have one row for an object and update the row whenever there are changes. However, the update operation is expensive and slow for a DBMS because it requires rewriting the data in the storage. Update is not acceptable if you need to write data quickly, but you can write the changes to an object sequentially as follows. -Use the particular column `Sign` when writing row. If `Sign = 1` it means that the row is a state of an object, let's call it "state" row. If `Sign = -1` it means the cancellation of the state of an object with the same attributes, let's call it "cancel" row. Also, use the particular column `Version` which should identify each state of an object with a separate number. +Use the `Sign` column when writing the row. If `Sign = 1` it means that the row is a state of an object (let's call it the "state" row). If `Sign = -1` it indicates the cancellation of the state of an object with the same attributes (let's call it the "cancel" row). Also use the `Version` column, which should identify each state of an object with a separate number. -For example, we want to calculate how much pages users checked at some site and how long they were there. At some moment of time we write the following row with the state of user activity: +For example, we want to calculate how many pages users visited on some site and how long they were there. At some point in time we write the following row with the state of user activity: ``` ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -87,7 +87,7 @@ For example, we want to calculate how much pages users checked at some site and └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -At some moment later we register the change of user activity and write it with the following two rows. +At some point later we register the change of user activity and write it with the following two rows. ``` ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -96,11 +96,11 @@ At some moment later we register the change of user activity and write it with t └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -The first row cancels the previous state of the object (user). It should copy all of the fields of the canceled state excepting `Sign`. +The first row cancels the previous state of the object (user). It should copy all of the fields of the canceled state except `Sign`. The second row contains the current state. -As we need only the last state of user activity, the rows +Because we need only the last state of user activity, the rows ``` ┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┬─Version─┐ @@ -109,31 +109,31 @@ As we need only the last state of user activity, the rows └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -can be deleted collapsing the invalid (old) state of an object. `VesionedCollapsingMergeTree` does this while merging of the data parts. +can be deleted, collapsing the invalid (old) state of the object. `VersionedCollapsingMergeTree` does this while merging the data parts. -Why we need 2 rows for each change read in the "Algorithm" paragraph. +To find out why we need two rows for each change, see "Algorithm". -**Peculiar properties of such approach** +**Notes on Usage** -1. The program that writes the data should remember the state of an object to be able to cancel it. "Cancel" string should be the copy of "state" string with the opposite `Sign`. It increases the initial size of storage but allows to write the data quickly. -2. Long growing arrays in columns reduce the efficiency of the engine due to load for writing. The more straightforward data, the higher efficiency. -3. `SELECT` results depend strongly on the consistency of object changes history. Be accurate when preparing data for inserting. You can get unpredictable results in inconsistent data, for example, negative values for non-negative metrics such as session depth. +1. The program that writes the data should remember the state of an object in order to cancel it. The "cancel" string should be a copy of the "state" string with the opposite `Sign`. This increases the initial size of storage but allows to write the data quickly. +2. Long growing arrays in columns reduce the efficiency of the engine due to the load for writing. The more straightforward the data, the better the efficiency. +3. `SELECT` results depend strongly on the consistency of the history of object changes. Be accurate when preparing data for inserting. You can get unpredictable results with inconsistent data, such as negative values for non-negative metrics like session depth. ### Algorithm -When ClickHouse merges data parts, it deletes each pair of rows having the same primary key and version and different `Sign`. The order of rows does not matter. +When ClickHouse merges data parts, it deletes each pair of rows that have the same primary key and version and different `Sign`. The order of rows does not matter. -When ClickHouse merges data parts, it orders rows by the primary key. If the `Version` column is not in the primary key, ClickHouse adds it to the primary key implicitly as the last field and use for ordering. +When ClickHouse inserts data, it orders rows by the primary key. If the `Version` column is not in the primary key, ClickHouse adds it to the primary key implicitly as the last field and uses it for ordering. ## Selecting Data -ClickHouse doesn't guarantee that all of the rows with the same primary key will be in the same resulting data part and even on the same physical server. It's true for writing the data and for subsequent merging of the data parts. Also, ClickHouse process `SELECT` queries with multiple threads, and it can not predict the order of rows in the result. So the aggregation is required if there is a need to get completely "collapsed" data from `VersionedCollapsingMergeTree` table. +ClickHouse doesn't guarantee that all of the rows with the same primary key will be in the same resulting data part or even on the same physical server. This is true both for writing the data and for subsequent merging of the data parts. In addition, ClickHouse processes `SELECT` queries with multiple threads, and it cannot predict the order of rows in the result. This means that aggregation is required if there is a need to get completely "collapsed" data from a `VersionedCollapsingMergeTree` table. -To finalize collapsing write a query with `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and so on, and also add `HAVING sum(Sign) > 0`. +To finalize collapsing, write a query with a `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and add `HAVING sum(Sign) > 0`. -The aggregates `count`, `sum` and `avg` could be calculated this way. The aggregate `uniq` could be calculated if an object has at list one state not collapsed. The aggregates `min` and `max` could not be calculated because `VersionedCollapsingMergeTree` does not save values history of the collapsed states. +The aggregates `count`, `sum` and `avg` can be calculated this way. The aggregate `uniq` can be calculated if an object has at least one non-collapsed state. The aggregates `min` and `max` can't be calculated because `VersionedCollapsingMergeTree` does not save the history of values of collapsed states. -If you need to extract the data with "collapsing" but without of aggregation (for example, to check whether rows are present whose newest values match certain conditions), you can use the `FINAL` modifier for the `FROM` clause. This approach is inefficient and should not be used with big tables. +If you need to extract the data with "collapsing" but without aggregation (for example, to check whether rows are present whose newest values match certain conditions), you can use the `FINAL` modifier for the `FROM` clause. This approach is inefficient and should not be used with large tables. ## Example of Use @@ -147,7 +147,7 @@ Example data: └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -Creation of the table: +Creating the table: ```sql CREATE TABLE UAct @@ -162,7 +162,7 @@ ENGINE = VersionedCollapsingMergeTree(Sign, Version) ORDER BY UserID ``` -Insertion of the data: +Inserting the data: ```sql INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) @@ -171,7 +171,7 @@ INSERT INTO UAct VALUES (4324182021466249494, 5, 146, 1, 1) INSERT INTO UAct VALUES (4324182021466249494, 5, 146, -1, 1),(4324182021466249494, 6, 185, 1, 2) ``` -We use two `INSERT` queries to create two different data parts. If we insert the data with one query ClickHouse creates one data part and will not perform any merge ever. +We use two `INSERT` queries to create two different data parts. If we insert the data with a single query, ClickHouse creates one data part and will never perform any merge. Getting the data: @@ -189,11 +189,11 @@ SELECT * FROM UAct └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -What do we see and where is collapsing? -With two `INSERT` queries, we created 2 data parts. The `SELECT` query have performed in 2 threads, and we got a random order of rows. -Collapsing not occurred because there was no merge of the data parts yet. ClickHouse merges data part in an unknown moment of time which we can not predict. +What do we see here and where are the collapsed parts? +We created two data parts using two `INSERT` queries. The `SELECT` query was performed in two threads, and the result is a random order of rows. +Collapsing did not occur because the data parts have not been merged yet. ClickHouse merges data parts at an unknown point in time which we cannot predict. -Thus we need aggregation: +This is why we need aggregation: ```sql SELECT @@ -211,7 +211,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┴─────────┘ ``` -If we do not need aggregation and want to force collapsing, we can use `FINAL` modifier for `FROM` clause. +If we don't need aggregation and want to force collapsing, we can use the `FINAL` modifier for the `FROM` clause. ```sql SELECT * FROM UAct FINAL @@ -222,6 +222,6 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -This way of selecting the data is very inefficient. Don't use it for big tables. +This is a very inefficient way to select data. Don't use it for large tables. [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/versionedcollapsingmergetree/) diff --git a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md b/docs/ru/operations/table_engines/versionedcollapsingmergetree.md deleted file mode 120000 index 5843fba70b8..00000000000 --- a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/operations/table_engines/versionedcollapsingmergetree.md \ No newline at end of file From a2ff9391ad0c099cd90263e0ff53c12804ca0730 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 14 Feb 2019 17:25:55 +0300 Subject: [PATCH 15/17] fix UB --- dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index febc9753366..35aa9461709 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -75,7 +75,8 @@ std::vector TranslateQualifiedNamesMatcher::visit(ASTIdentifier & iden /// In case if column from the joined table are in source columns, change it's name to qualified. if (best_table_pos && data.source_columns.count(identifier.shortName())) IdentifierSemantic::setNeedLongName(identifier, true); - IdentifierSemantic::setColumnNormalName(identifier, data.tables[best_table_pos].first); + if (!data.tables.empty()) + IdentifierSemantic::setColumnNormalName(identifier, data.tables[best_table_pos].first); } return {}; From 6cf3d8c15176c3209f9039502166fe1757ca59ae Mon Sep 17 00:00:00 2001 From: Vivien Maisonneuve Date: Thu, 14 Feb 2019 17:44:46 +0100 Subject: [PATCH 16/17] Doc fix: definition of varPop (#4403) --- docs/en/query_language/agg_functions/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index 004a8176fc9..30e571d5fba 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -421,7 +421,7 @@ Returns `Float64`. When `n <= 1`, returns `+∞`. ## varPop(x) -Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. +Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. From 81f8d2f1f31c99ae904ae74b5105e549842005db Mon Sep 17 00:00:00 2001 From: Murat Kabilov Date: Fri, 15 Feb 2019 09:45:22 +0100 Subject: [PATCH 17/17] add columns parameter to the syntax for SummingMergeTree (#4400) --- docs/en/operations/table_engines/summingmergetree.md | 2 +- docs/ru/operations/table_engines/summingmergetree.md | 2 +- docs/zh/operations/table_engines/summingmergetree.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/table_engines/summingmergetree.md b/docs/en/operations/table_engines/summingmergetree.md index 32444b61c1a..a5458a5a1e0 100644 --- a/docs/en/operations/table_engines/summingmergetree.md +++ b/docs/en/operations/table_engines/summingmergetree.md @@ -13,7 +13,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE = SummingMergeTree() +) ENGINE = SummingMergeTree([columns]) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] diff --git a/docs/ru/operations/table_engines/summingmergetree.md b/docs/ru/operations/table_engines/summingmergetree.md index b10d1598fb8..9ca1052fe5e 100644 --- a/docs/ru/operations/table_engines/summingmergetree.md +++ b/docs/ru/operations/table_engines/summingmergetree.md @@ -13,7 +13,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE = SummingMergeTree() +) ENGINE = SummingMergeTree([columns]) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] diff --git a/docs/zh/operations/table_engines/summingmergetree.md b/docs/zh/operations/table_engines/summingmergetree.md index d1e35124fe0..82664cc2d9b 100644 --- a/docs/zh/operations/table_engines/summingmergetree.md +++ b/docs/zh/operations/table_engines/summingmergetree.md @@ -13,7 +13,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) ENGINE = SummingMergeTree() +) ENGINE = SummingMergeTree([columns]) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr]