From cbeda6c60e6fdf90803636844aa4dd18d94f1e3e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Feb 2021 23:04:45 +0300 Subject: [PATCH 1/7] Fix LOGICAL_ERROR for join_use_nulls=1 when JOIN contains const from SELECT --- src/Interpreters/TableJoin.cpp | 10 +++++++++- tests/queries/0_stateless/01710_join_use_nulls.sql | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 2d3bffa8234..c1777711d9e 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -230,8 +230,16 @@ void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column) void TableJoin::addJoinedColumnsAndCorrectNullability(ColumnsWithTypeAndName & columns) const { for (auto & col : columns) + { if (leftBecomeNullable(col.type)) - col.type = makeNullable(col.type); + { + /// No need to nullify constants + if (!(col.column && isColumnConst(*col.column))) + { + col.type = makeNullable(col.type); + } + } + } for (const auto & col : columns_added_by_join) { diff --git a/tests/queries/0_stateless/01710_join_use_nulls.sql b/tests/queries/0_stateless/01710_join_use_nulls.sql index 2845af8b8ed..5486010183a 100644 --- a/tests/queries/0_stateless/01710_join_use_nulls.sql +++ b/tests/queries/0_stateless/01710_join_use_nulls.sql @@ -11,5 +11,11 @@ FROM X RIGHT JOIN Y ON (X.id + 1) = Y.id SETTINGS join_use_nulls=1; -- { serverError 53 } +-- Logical error: 'Arguments of 'plus' have incorrect data types: '2' of type 'UInt8', '1' of type 'UInt8''. +-- Because 1 became toNullable(1), i.e.: +-- 2 UInt8 Const(size = 1, UInt8(size = 1)) +-- 1 UInt8 Const(size = 1, Nullable(size = 1, UInt8(size = 1), UInt8(size = 1))) +SELECT 2+1 FROM system.one X RIGHT JOIN system.one Y ON X.dummy+1 = Y.dummy SETTINGS join_use_nulls = 1; -- { serverError 53 } + DROP TABLE X; DROP TABLE Y; From 4aa46ce3d60007819ffc43b674bbb8e4fdf75df7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Feb 2021 10:46:29 +0300 Subject: [PATCH 2/7] More tests for join_use_nulls All of them already works, but just in case --- .../queries/0_stateless/01710_join_use_nulls.reference | 3 +++ tests/queries/0_stateless/01710_join_use_nulls.sql | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01710_join_use_nulls.reference b/tests/queries/0_stateless/01710_join_use_nulls.reference index e69de29bb2d..8bd111e0416 100644 --- a/tests/queries/0_stateless/01710_join_use_nulls.reference +++ b/tests/queries/0_stateless/01710_join_use_nulls.reference @@ -0,0 +1,3 @@ +3 +1 +1 diff --git a/tests/queries/0_stateless/01710_join_use_nulls.sql b/tests/queries/0_stateless/01710_join_use_nulls.sql index 5486010183a..b024227d4e2 100644 --- a/tests/queries/0_stateless/01710_join_use_nulls.sql +++ b/tests/queries/0_stateless/01710_join_use_nulls.sql @@ -5,17 +5,17 @@ CREATE TABLE X (id Int) ENGINE=Memory; CREATE TABLE Y (id Int) ENGINE=Memory; -- Type mismatch of columns to JOIN by: plus(id, 1) Int64 at left, Y.id Int32 at right. -SELECT - Y.id - 1 -FROM X -RIGHT JOIN Y ON (X.id + 1) = Y.id -SETTINGS join_use_nulls=1; -- { serverError 53 } +SELECT Y.id - 1 FROM X RIGHT JOIN Y ON (X.id + 1) = Y.id SETTINGS join_use_nulls=1; -- { serverError 53 } +SELECT Y.id - 1 FROM X RIGHT JOIN Y ON (X.id + 1) = toInt64(Y.id) SETTINGS join_use_nulls=1; -- Logical error: 'Arguments of 'plus' have incorrect data types: '2' of type 'UInt8', '1' of type 'UInt8''. -- Because 1 became toNullable(1), i.e.: -- 2 UInt8 Const(size = 1, UInt8(size = 1)) -- 1 UInt8 Const(size = 1, Nullable(size = 1, UInt8(size = 1), UInt8(size = 1))) SELECT 2+1 FROM system.one X RIGHT JOIN system.one Y ON X.dummy+1 = Y.dummy SETTINGS join_use_nulls = 1; -- { serverError 53 } +SELECT 2+1 FROM system.one X RIGHT JOIN system.one Y ON X.dummy+1 = toUInt16(Y.dummy) SETTINGS join_use_nulls = 1; +SELECT X.dummy+1 FROM system.one X RIGHT JOIN system.one Y ON X.dummy = Y.dummy SETTINGS join_use_nulls = 1; +SELECT Y.dummy+1 FROM system.one X RIGHT JOIN system.one Y ON X.dummy = Y.dummy SETTINGS join_use_nulls = 1; DROP TABLE X; DROP TABLE Y; From 179a0f9d8bfd540e730abacbe9c11d945ac3b405 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Feb 2021 00:26:25 +0300 Subject: [PATCH 3/7] Performance improvement by Nikolai Kochetov --- src/Storages/StorageMemory.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index dc695427156..79ced856231 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -45,6 +45,8 @@ public: /// Smaller blocks (e.g. 64K rows) are better for CPU cache. bool prefersLargeBlocks() const override { return false; } + bool hasEvenlyDistributedRead() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, const Context & context) override; void drop() override; From 17dce001362e9a178681756ae0498ef36b134008 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 14 Feb 2021 10:45:52 +0300 Subject: [PATCH 4/7] Temporary disable 00992_system_parts_race_condition_zookeeper --- tests/queries/skip_list.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index e4e7504ba41..ee25bee6a0a 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -10,6 +10,7 @@ "00152_insert_different_granularity", "00151_replace_partition_with_different_granularity", "00157_cache_dictionary", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01193_metadata_loading", "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers @@ -25,6 +26,7 @@ "memory_profiler", "odbc_roundtrip", "01103_check_cpu_instructions_at_startup", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -35,6 +37,7 @@ "memory_profiler", "01103_check_cpu_instructions_at_startup", "00900_orc_load", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -46,6 +49,7 @@ "01103_check_cpu_instructions_at_startup", "01086_odbc_roundtrip", /// can't pass because odbc libraries are not instrumented "00877_memory_limit_for_new_delete", /// memory limits don't work correctly under msan because it replaces malloc/free + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -57,6 +61,7 @@ "00980_alter_settings_race", "00834_kill_mutation_replicated_zookeeper", "00834_kill_mutation", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01200_mutations_memory_consumption", "01103_check_cpu_instructions_at_startup", "01037_polygon_dicts_", @@ -82,6 +87,7 @@ "00505_secure", "00505_shard_secure", "odbc_roundtrip", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01103_check_cpu_instructions_at_startup", "01114_mysql_database_engine_segfault", "00834_cancel_http_readonly_queries_on_client_close", @@ -95,16 +101,19 @@ "01455_time_zones" ], "release-build": [ + "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) ], "database-ordinary": [ "00604_show_create_database", "00609_mv_index_in_in", "00510_materizlized_view_and_deduplication_zookeeper", - "00738_lock_for_inner_table" + "00738_lock_for_inner_table", + "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) ], "polymorphic-parts": [ "01508_partition_pruning_long", /// bug, shoud be fixed - "01482_move_to_prewhere_and_cast" /// bug, shoud be fixed + "01482_move_to_prewhere_and_cast", /// bug, shoud be fixed + "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) ], "antlr": [ "00186_very_long_arrays", @@ -144,6 +153,7 @@ "00982_array_enumerate_uniq_ranked", "00984_materialized_view_to_columns", "00988_constraints_replication_zookeeper", + "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "00995_order_by_with_fill", "01001_enums_in_in_section", "01011_group_uniq_array_memsan", From 607b57ea2842fee07a3a20c42f0b4aabc9623186 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 14 Feb 2021 10:57:52 +0300 Subject: [PATCH 5/7] Update version_date.tsv after release 21.2.3.15 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 8d05f5fff46..f4616027512 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v21.2.3.15-stable 2021-02-14 v21.2.2.8-stable 2021-02-07 v21.1.3.32-stable 2021-02-03 v21.1.2.15-stable 2021-01-18 From 09a5b7a05535b7fd5725bd80f5f13ad9bf05de7a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 14 Feb 2021 11:35:34 +0300 Subject: [PATCH 6/7] Update version_date.tsv after release 21.1.4.46 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f4616027512..43a1b3eba50 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,5 +1,6 @@ v21.2.3.15-stable 2021-02-14 v21.2.2.8-stable 2021-02-07 +v21.1.4.46-stable 2021-02-14 v21.1.3.32-stable 2021-02-03 v21.1.2.15-stable 2021-01-18 v20.12.5.18-stable 2021-02-03 From 37807e1a18a3bef186b97eb845faa943fa98f537 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 14 Feb 2021 11:51:46 +0300 Subject: [PATCH 7/7] Update version_date.tsv after release 20.12.6.29 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 43a1b3eba50..d0d782e77ec 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -3,6 +3,7 @@ v21.2.2.8-stable 2021-02-07 v21.1.4.46-stable 2021-02-14 v21.1.3.32-stable 2021-02-03 v21.1.2.15-stable 2021-01-18 +v20.12.6.29-stable 2021-02-14 v20.12.5.18-stable 2021-02-03 v20.12.5.14-stable 2020-12-28 v20.12.4.5-stable 2020-12-24