From 60d63081da829383b6f90931447ff1b1d07c974c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 28 Jan 2020 23:10:06 +0300 Subject: [PATCH 1/5] Fix min_marks_for_seek for the data skipping indexes --- dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 09c4fe835d6..b50dd7146da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1272,8 +1272,8 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( const size_t min_marks_for_seek = roundRowsOrBytesToMarks( settings.merge_tree_min_rows_for_seek, settings.merge_tree_min_bytes_for_seek, - part->index_granularity_info.index_granularity_bytes, - part->index_granularity_info.fixed_index_granularity); + part->index_granularity_info.fixed_index_granularity, + part->index_granularity_info.index_granularity_bytes); size_t granules_dropped = 0; From c6d95e9dfd3c6cffc604a30da5b49f6076b498c4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Jan 2020 23:31:19 +0300 Subject: [PATCH 2/5] Rename tables to match the test number for 01071_force_optimize_skip_unused_shards --- ...1071_force_optimize_skip_unused_shards.sql | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql b/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql index f12d5f8846d..8a2ed3ca0f9 100644 --- a/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql +++ b/dbms/tests/queries/0_stateless/01071_force_optimize_skip_unused_shards.sql @@ -1,26 +1,26 @@ set optimize_skip_unused_shards=1; -drop table if exists data_01068; -drop table if exists dist_01068; +drop table if exists data_01071; +drop table if exists dist_01071; -create table data_01068 (key Int) Engine=Null(); +create table data_01071 (key Int) Engine=Null(); -create table dist_01068 as data_01068 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01068); +create table dist_01071 as data_01071 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01071); set force_optimize_skip_unused_shards=0; -select * from dist_01068; +select * from dist_01071; set force_optimize_skip_unused_shards=1; -select * from dist_01068; +select * from dist_01071; set force_optimize_skip_unused_shards=2; -select * from dist_01068; -- { serverError 507 } +select * from dist_01071; -- { serverError 507 } -drop table if exists dist_01068; -create table dist_01068 as data_01068 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01068, key%2); +drop table if exists dist_01071; +create table dist_01071 as data_01071 Engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01071, key%2); set force_optimize_skip_unused_shards=0; -select * from dist_01068; +select * from dist_01071; set force_optimize_skip_unused_shards=1; -select * from dist_01068; -- { serverError 507 } +select * from dist_01071; -- { serverError 507 } set force_optimize_skip_unused_shards=2; -select * from dist_01068; -- { serverError 507 } +select * from dist_01071; -- { serverError 507 } -drop table if exists data_01068; -drop table if exists dist_01068; +drop table if exists data_01071; +drop table if exists dist_01071; From ca8d9ac7f8ad0f629a8c89b20c51187d825c02a6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Jan 2020 23:08:49 +0300 Subject: [PATCH 3/5] Fix error message for force_optimize_skip_unused_shards --- dbms/src/Storages/StorageDistributed.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index ddf727c21de..3dabc4aa701 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -399,7 +399,7 @@ BlockInputStreams StorageDistributed::read( if (force) { std::stringstream exception_message; - if (has_sharding_key) + if (!has_sharding_key) exception_message << "No sharding key"; else exception_message << "Sharding key " << sharding_key_column_name << " is not used"; From fbbf0262773a4a42cb766dd19dd580a6afaf14bf Mon Sep 17 00:00:00 2001 From: Baudouin Giard Date: Fri, 31 Jan 2020 14:31:11 +0000 Subject: [PATCH 4/5] String keys in SummingMergeTree maps Accept String or FixedString data types as key for maps in the SummingMergeTree now that sumMap supports these two types (introduced in bc0fbd688a26a7dbf1927d568f44c2e46d2ab8e8). Signed-off-by: Baudouin Giard --- .../DataStreams/SummingSortedBlockInputStream.cpp | 2 +- .../00146_summing_merge_tree_nested_map.reference | 4 ++++ .../00146_summing_merge_tree_nested_map.sql | 9 +++++++++ .../00327_summing_composite_nested.reference | 14 +++++++------- .../0_stateless/00327_summing_composite_nested.sql | 4 ++-- .../operations/table_engines/summingmergetree.md | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp index fe29dc55916..8cd1bd3071e 100644 --- a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp @@ -156,7 +156,7 @@ SummingSortedBlockInputStream::SummingSortedBlockInputStream( || endsWith(name, "Key") || endsWith(name, "Type")) { - if (!nested_type.isValueRepresentedByInteger()) + if (!nested_type.isValueRepresentedByInteger() && !isStringOrFixedString(nested_type)) break; map_desc.key_col_nums.push_back(*column_num_it); diff --git a/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.reference b/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.reference index ebe1ea57d7f..93d33b516d1 100644 --- a/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.reference +++ b/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.reference @@ -2,6 +2,10 @@ [1] [250] [1,2] [250,150] [2] [150] +['1','2'] [100,150] +['1'] [250] +['1','2'] [250,150] +['2'] [150] 20 [1,2] [100,150] 20 [1] [250] 20 [1,2] [250,150] diff --git a/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql b/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql index fac33033bfb..ab9e8c9b177 100644 --- a/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql +++ b/dbms/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql @@ -9,6 +9,15 @@ select `SomeMap.ID`, `SomeMap.Num` from nested_map; drop table nested_map; +create table nested_map (d default today(), k UInt64, payload default rand(), SomeMap Nested(ID String, Num Int64)) engine=SummingMergeTree(d, k, 8192); + +insert into nested_map (k, `SomeMap.ID`, `SomeMap.Num`) values (0,['1'],[100]),(1,['1'],[100]),(2,['1'],[100]),(3,['1','2'],[100,150]); +insert into nested_map (k, `SomeMap.ID`, `SomeMap.Num`) values (0,['2'],[150]),(1,['1'],[150]),(2,['1','2'],[150,150]),(3,['1'],[-100]); +optimize table nested_map; +select `SomeMap.ID`, `SomeMap.Num` from nested_map; + +drop table nested_map; + drop table if exists nested_map_explicit; create table nested_map_explicit (d default today(), k UInt64, SomeIntExcluded UInt32, SomeMap Nested(ID UInt32, Num Int64)) engine=SummingMergeTree(d, k, 8192, (SomeMap)); diff --git a/dbms/tests/queries/0_stateless/00327_summing_composite_nested.reference b/dbms/tests/queries/0_stateless/00327_summing_composite_nested.reference index 93ed05ad21e..b4233202447 100644 --- a/dbms/tests/queries/0_stateless/00327_summing_composite_nested.reference +++ b/dbms/tests/queries/0_stateless/00327_summing_composite_nested.reference @@ -1,8 +1,8 @@ -2000-01-01 1 [1,2] [3,4] [10,11] [0,1,2] [3,4,5] [-1,-2,-3] [1,10,100] -2000-01-01 1 [2,1] [4,3] [20,22] [2,2,1] [5,5,0] [-3,-3,-33] [10,100,1000] -2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] [3,4,5] [-1,-2,-3] [1,10,100] -2000-01-01 2 [2,1,1] [4,3,3] [20,22,33] [2,2] [5,5] [-3,-3] [10,100] -2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] [3,4,5] [-1,-2,-3] [1,10,100] +2000-01-01 1 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] +2000-01-01 1 [2,1] [4,3] [20,22] [2,2,1] ['5','5','0'] [-3,-3,-33] [10,100,1000] +2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] +2000-01-01 2 [2,1,1] [4,3,3] [20,22,33] [2,2] ['5','5'] [-3,-3] [10,100] +2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] 2000-01-01 1 1 3 10 2000-01-01 1 1 3 22 2000-01-01 1 2 4 11 @@ -50,8 +50,8 @@ 2000-01-01 2 0 3 -1 2 2000-01-01 2 1 4 -2 20 2000-01-01 2 2 5 -3 310 -2000-01-01 1 [1,2] [3,4] [32,31] [0,1,1,2] [3,0,4,5] [-1,-33,-2,-3] [1,1000,10,210] -2000-01-01 2 [1,2] [3,4] [75,42] [0,1,2] [3,4,5] [-1,-2,-3] [2,20,310] +2000-01-01 1 [1,2] [3,4] [32,31] [0,1,1,2] ['3','0','4','5'] [-1,-33,-2,-3] [1,1000,10,210] +2000-01-01 2 [1,2] [3,4] [75,42] [0,1,2] ['3','4','5'] [-1,-2,-3] [2,20,310] 2000-01-01 1 1 3 32 2000-01-01 1 2 4 31 2000-01-01 2 1 3 75 diff --git a/dbms/tests/queries/0_stateless/00327_summing_composite_nested.sql b/dbms/tests/queries/0_stateless/00327_summing_composite_nested.sql index 73efd231a83..b61bc71b892 100644 --- a/dbms/tests/queries/0_stateless/00327_summing_composite_nested.sql +++ b/dbms/tests/queries/0_stateless/00327_summing_composite_nested.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS summing_composite_key; -CREATE TABLE summing_composite_key (d Date, k UInt64, FirstMap Nested(k1 UInt32, k2ID Int8, s Float64), SecondMap Nested(k1ID UInt64, k2Key UInt32, k3Type Int32, s Int64)) ENGINE = SummingMergeTree(d, k, 1); +CREATE TABLE summing_composite_key (d Date, k UInt64, FirstMap Nested(k1 UInt32, k2ID Int8, s Float64), SecondMap Nested(k1ID UInt64, k2Key String, k3Type Int32, s Int64)) ENGINE = SummingMergeTree(d, k, 1); -INSERT INTO summing_composite_key VALUES ('2000-01-01', 1, [1,2], [3,4], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 1, [2,1], [4,3], [20,22], [2,2,1], [5,5,0], [-3,-3,-33], [10,100,1000]), ('2000-01-01', 2, [1,2], [3,4], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 2, [2,1,1], [4,3,3], [20,22,33], [2,2], [5,5], [-3,-3], [10,100]), ('2000-01-01', 2, [1,2], [3,4], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]); +INSERT INTO summing_composite_key VALUES ('2000-01-01', 1, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 1, [2,1], ['4','3'], [20,22], [2,2,1], [5,5,0], [-3,-3,-33], [10,100,1000]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 2, [2,1,1], ['4','3','3'], [20,22,33], [2,2], [5,5], [-3,-3], [10,100]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]); SELECT * FROM summing_composite_key ORDER BY d, k, _part_index; diff --git a/docs/en/operations/table_engines/summingmergetree.md b/docs/en/operations/table_engines/summingmergetree.md index 332daa60a92..bb66d1b7a22 100644 --- a/docs/en/operations/table_engines/summingmergetree.md +++ b/docs/en/operations/table_engines/summingmergetree.md @@ -112,7 +112,7 @@ Table can have nested data structures that are processed in a special way. If the name of a nested table ends with `Map` and it contains at least two columns that meet the following criteria: -- the first column is numeric `(*Int*, Date, DateTime)`, let's call it `key`, +- the first column is numeric `(*Int*, Date, DateTime)` or a string `(String, FixedString)`, let's call it `key`, - the other columns are arithmetic `(*Int*, Float32/64)`, let's call it `(values...)`, then this nested table is interpreted as a mapping of `key => (values...)`, and when merging its rows, the elements of two data sets are merged by `key` with a summation of the corresponding `(values...)`. From 23dedcbfa974a068a1993ecc9d6e65cd2d81f82e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 31 Jan 2020 20:45:06 +0300 Subject: [PATCH 5/5] Update robots.txt --- website/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/website/robots.txt b/website/robots.txt index 300b2e7da6e..2af539491b1 100644 --- a/website/robots.txt +++ b/website/robots.txt @@ -12,6 +12,7 @@ Disallow: /docs/ru/search.html Disallow: /docs/ja/search.html Disallow: /docs/zh/search.html Disallow: /docs/fa/search.html +Disallow: /cdn-cgi/ Allow: / Host: https://clickhouse.tech Sitemap: https://clickhouse.tech/docs/sitemap.xml