From b9f49d31dff96f8883f7691095fda628bd77fec3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 30 Jul 2020 22:08:13 +0300 Subject: [PATCH 01/10] Sanity checks for MergeTreeSettings --- programs/server/Server.cpp | 3 +++ src/Storages/MergeTree/MergeTreeData.cpp | 4 +++ src/Storages/MergeTree/MergeTreeSettings.cpp | 27 +++++++++++++++++++ src/Storages/MergeTree/MergeTreeSettings.h | 4 +++ ...merge_tree_settings_sanity_check.reference | 0 ...01419_merge_tree_settings_sanity_check.sql | 21 +++++++++++++++ 6 files changed, 59 insertions(+) create mode 100644 tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.reference create mode 100644 tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c3b17824151..c0bb1746cdb 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -641,6 +641,9 @@ int Server::main(const std::vector & /*args*/) global_context->setFormatSchemaPath(format_schema_path.path()); format_schema_path.createDirectories(); + /// Check sanity of MergeTreeSettings on server startup + global_context->getMergeTreeSettings().sanityCheck(settings); + /// Limit on total memory usage size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 740d44605ee..7f5773024ca 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -147,6 +147,10 @@ MergeTreeData::MergeTreeData( if (relative_data_path.empty()) throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME); + /// Check sanity of MergeTreeSettings. Only when table is created. + if (!attach) + settings->sanityCheck(global_context.getSettingsRef()); + MergeTreeDataFormatVersion min_format_version(0); if (!date_column_name.empty()) { diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 5c4113c1565..ff5d15bc3f1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -76,4 +76,31 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) #undef ADD_IF_ABSENT } +void MergeTreeSettings::sanityCheck(const Settings & query_settings) const +{ + if (number_of_free_entries_in_pool_to_execute_mutation >= query_settings.background_pool_size) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_execute_mutation' setting" + " ({}) (default values are defined in section of config.xml" + " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" + " is greater or equals to the value of 'background_pool_size'" + " ({}) (the value is defined in users.xml for default profile)." + " This indicates incorrect configuration because mutations cannot work with these settings.", + number_of_free_entries_in_pool_to_execute_mutation, + query_settings.background_pool_size); + } + + if (number_of_free_entries_in_pool_to_lower_max_size_of_merge >= query_settings.background_pool_size) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting" + " ({}) (default values are defined in section of config.xml" + " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" + " is greater or equals to the value of 'background_pool_size'" + " ({}) (the value is defined in users.xml for default profile)." + " This indicates incorrect configuration because the maximum size of merge will be always lowered.", + number_of_free_entries_in_pool_to_execute_mutation, + query_settings.background_pool_size); + } +} + } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 833425ff592..e8c817895d1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -18,6 +18,7 @@ namespace DB { class ASTStorage; +struct Settings; /** Settings for the MergeTree family of engines. * Could be loaded from config or from a CREATE TABLE query (SETTINGS clause). @@ -127,6 +128,9 @@ struct MergeTreeSettings : public SettingsCollection return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part" || name == "min_bytes_for_compact_part" || name == "min_rows_for_compact_part"; } + + /// Check that the values are sane taking also query-level settings into account. + void sanityCheck(const Settings & query_settings) const; }; using MergeTreeSettingsPtr = std::shared_ptr; diff --git a/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.reference b/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql b/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql new file mode 100644 index 00000000000..525fff9b256 --- /dev/null +++ b/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql @@ -0,0 +1,21 @@ +CREATE TABLE mytable_local +( + created DateTime, + eventday Date, + user_id UInt32 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(eventday) +ORDER BY (eventday, user_id) +SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 100; -- { serverError 36 } + +CREATE TABLE mytable_local +( + created DateTime, + eventday Date, + user_id UInt32 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(eventday) +ORDER BY (eventday, user_id) +SETTINGS number_of_free_entries_in_pool_to_lower_max_size_of_merge = 100; -- { serverError 36 } From 2c40539df64453daac914f914f74a3d688e5e3d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 31 Jul 2020 00:42:55 +0300 Subject: [PATCH 02/10] Check ALTERs; update test --- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++++- .../01415_inconsistent_merge_tree_settings.sql | 2 +- .../01419_merge_tree_settings_sanity_check.sql | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7f5773024ca..d1d4dae59a7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1471,7 +1471,6 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S if (old_metadata.hasSettingsChanges()) { - const auto current_changes = old_metadata.getSettingsChanges()->as().changes; const auto & new_changes = new_metadata.settings_changes->as().changes; for (const auto & changed_setting : new_changes) @@ -1619,6 +1618,7 @@ void MergeTreeData::changeSettings( const auto & new_changes = new_settings->as().changes; for (const auto & change : new_changes) + { if (change.name == "storage_policy") { StoragePolicyPtr new_storage_policy = global_context.getStoragePolicy(change.value.safeGet()); @@ -1653,9 +1653,13 @@ void MergeTreeData::changeSettings( has_storage_policy_changed = true; } } + } MergeTreeSettings copy = *getSettings(); copy.applyChanges(new_changes); + + copy.sanityCheck(global_context.getSettingsRef()); + storage_settings.set(std::make_unique(copy)); StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); new_metadata.setSettingsChanges(new_settings); diff --git a/tests/queries/0_stateless/01415_inconsistent_merge_tree_settings.sql b/tests/queries/0_stateless/01415_inconsistent_merge_tree_settings.sql index f3bf24193a8..2ce0575c4ad 100644 --- a/tests/queries/0_stateless/01415_inconsistent_merge_tree_settings.sql +++ b/tests/queries/0_stateless/01415_inconsistent_merge_tree_settings.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS t; SET mutations_sync = 1; -CREATE TABLE t (x UInt8, s String) ENGINE = MergeTree ORDER BY x SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 1000; +CREATE TABLE t (x UInt8, s String) ENGINE = MergeTree ORDER BY x SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 15; INSERT INTO t VALUES (1, 'hello'); SELECT * FROM t; diff --git a/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql b/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql index 525fff9b256..686594f435d 100644 --- a/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql +++ b/tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql @@ -1,3 +1,5 @@ +DROP TABLE IF EXISTS mytable_local; + CREATE TABLE mytable_local ( created DateTime, @@ -19,3 +21,17 @@ ENGINE = MergeTree() PARTITION BY toYYYYMM(eventday) ORDER BY (eventday, user_id) SETTINGS number_of_free_entries_in_pool_to_lower_max_size_of_merge = 100; -- { serverError 36 } + +CREATE TABLE mytable_local +( + created DateTime, + eventday Date, + user_id UInt32 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(eventday) +ORDER BY (eventday, user_id); + +ALTER TABLE mytable_local MODIFY SETTING number_of_free_entries_in_pool_to_execute_mutation = 100; -- { serverError 36 } + +DROP TABLE mytable_local; From b232659439c0e4fe07fc8f3d1eea49f15692cb17 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Aug 2020 00:39:51 +0300 Subject: [PATCH 03/10] Update tests --- .../configs/forbid_background_merges.xml | 3 ++- .../configs/forbid_background_merges.xml | 3 ++- .../test_polymorphic_parts/configs/do_not_merge.xml | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml b/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml index ffdca0cf6bc..f1f3af4224e 100644 --- a/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml +++ b/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml @@ -1,7 +1,8 @@ - 0 + 1 + 2 diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml index ffdca0cf6bc..f1f3af4224e 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml @@ -1,7 +1,8 @@ - 0 + 1 + 2 diff --git a/tests/integration/test_polymorphic_parts/configs/do_not_merge.xml b/tests/integration/test_polymorphic_parts/configs/do_not_merge.xml index 8b57af4f48e..82aaeb1fbc8 100644 --- a/tests/integration/test_polymorphic_parts/configs/do_not_merge.xml +++ b/tests/integration/test_polymorphic_parts/configs/do_not_merge.xml @@ -2,7 +2,6 @@ 1 2 - 100 0 From eb2236fe8bbd49e7cf67106a6db6526b2505acd1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 3 Aug 2020 14:13:43 +0300 Subject: [PATCH 04/10] Fix configs --- .../configs/forbid_background_merges.xml | 10 ++++------ tests/integration/test_insert_into_distributed/test.py | 2 +- .../configs/forbid_background_merges.xml | 10 ++++------ .../test.py | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml b/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml index f1f3af4224e..bc2dae31ad6 100644 --- a/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml +++ b/tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml @@ -1,8 +1,6 @@ - - - 1 - 2 - - + + 1 + 2 + diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index e48584bac84..731ffbbe2fd 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -12,7 +12,7 @@ instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_c instance_test_inserts_batching = cluster.add_instance( 'instance_test_inserts_batching', main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml']) -remote = cluster.add_instance('remote', user_configs=['configs/forbid_background_merges.xml']) +remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml']) instance_test_inserts_local_cluster = cluster.add_instance( 'instance_test_inserts_local_cluster', diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml index f1f3af4224e..bc2dae31ad6 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml @@ -1,8 +1,6 @@ - - - 1 - 2 - - + + 1 + 2 + diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py index 2dc8d7326dd..1df803920f1 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py @@ -12,7 +12,7 @@ instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_c instance_test_inserts_batching = cluster.add_instance( 'instance_test_inserts_batching', main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml']) -remote = cluster.add_instance('remote', user_configs=['configs/forbid_background_merges.xml']) +remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml']) instance_test_inserts_local_cluster = cluster.add_instance( 'instance_test_inserts_local_cluster', From 41f30d3bd8b11ef94eec15a2531035677b5691c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Aug 2020 17:32:14 +0300 Subject: [PATCH 05/10] Fix range checks in h3 functions --- src/Functions/h3ToChildren.cpp | 8 ++++++++ src/Functions/h3ToParent.cpp | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 6c1b685bcd9..a3dea614fa6 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -7,15 +7,19 @@ #include #include +#include #include namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; } + class FunctionH3ToChildren : public IFunction { public: @@ -63,6 +67,10 @@ public: const UInt64 parent_hindex = col_hindex->getUInt(row); const UInt8 child_resolution = col_resolution->getUInt(row); + if (child_resolution > MAX_H3_RES) + throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName() + + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + const auto vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution); hindex_vec.resize(vec_size); h3ToChildren(parent_hindex, child_resolution, hindex_vec.data()); diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 981628ae8f8..90e38c2ee27 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -5,15 +5,19 @@ #include #include +#include #include namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; } + class FunctionH3ToParent : public IFunction { public: @@ -57,6 +61,10 @@ public: const UInt64 hindex = col_hindex->getUInt(row); const UInt8 resolution = col_resolution->getUInt(row); + if (resolution > MAX_H3_RES) + throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() + + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + UInt64 res = h3ToParent(hindex, resolution); dst_data[row] = res; From 0e515aa1c9cf8d5d30c72bd1aee1c469825b7675 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Aug 2020 17:49:50 +0300 Subject: [PATCH 06/10] Fixup --- src/Functions/h3ToChildren.cpp | 1 + src/Functions/h3ToParent.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index a3dea614fa6..926028ce219 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 90e38c2ee27..2f6a9f3264d 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include From 7bd0620deb5fb5ac984f0abbc8fbe8007d1521c2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Aug 2020 19:39:08 +0300 Subject: [PATCH 07/10] Added another check just in case --- src/Functions/h3ToChildren.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 926028ce219..d9e402231f7 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -12,6 +12,9 @@ #include +static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; + + namespace DB { @@ -19,6 +22,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int TOO_LARGE_ARRAY_SIZE; } class FunctionH3ToChildren : public IFunction @@ -72,7 +76,12 @@ public: throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName() + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - const auto vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution); + const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution); + if (vec_size > MAX_ARRAY_SIZE) + throw Exception("The result of function" + getName() + + " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = " + + toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE); + hindex_vec.resize(vec_size); h3ToChildren(parent_hindex, child_resolution, hindex_vec.data()); From 477852cde9d097ee0e3fdda2f9e00dc9af50a96d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Aug 2020 19:41:10 +0300 Subject: [PATCH 08/10] Add test --- tests/queries/0_stateless/01428_h3_range_check.reference | 0 tests/queries/0_stateless/01428_h3_range_check.sql | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/01428_h3_range_check.reference create mode 100644 tests/queries/0_stateless/01428_h3_range_check.sql diff --git a/tests/queries/0_stateless/01428_h3_range_check.reference b/tests/queries/0_stateless/01428_h3_range_check.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01428_h3_range_check.sql b/tests/queries/0_stateless/01428_h3_range_check.sql new file mode 100644 index 00000000000..7c7376a90ae --- /dev/null +++ b/tests/queries/0_stateless/01428_h3_range_check.sql @@ -0,0 +1,2 @@ +SELECT h3ToChildren(599405990164561919, 100); -- { serverError 69 } +SELECT h3ToParent(599405990164561919, 100); -- { serverError 69 } From 65c755190e9ac650cd07939035255c56cb250f33 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 4 Aug 2020 01:18:59 +0300 Subject: [PATCH 09/10] =?UTF-8?q?Revert=20"Add=20QueryTimeMicroseconds,=20?= =?UTF-8?q?SelectQueryTimeMicroseconds=20and=20InsertQuer=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Common/ProfileEvents.cpp | 3 -- src/Interpreters/executeQuery.cpp | 54 +++++++------------ ...1417_query_time_in_system_events.reference | 3 -- .../01417_query_time_in_system_events.sh | 51 ------------------ 4 files changed, 20 insertions(+), 91 deletions(-) delete mode 100644 tests/queries/0_stateless/01417_query_time_in_system_events.reference delete mode 100755 tests/queries/0_stateless/01417_query_time_in_system_events.sh diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 475e073d253..908adbe1d12 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -11,9 +11,6 @@ M(FailedQuery, "Number of failed queries.") \ M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \ M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.") \ - M(QueryTimeMicroseconds, "Total time of all queries.") \ - M(SelectQueryTimeMicroseconds, "Total time of SELECT queries.") \ - M(InsertQueryTimeMicroseconds, "Total time of INSERT queries.") \ M(FileOpen, "Number of files opened.") \ M(Seek, "Number of times the 'lseek' function was called.") \ M(ReadBufferFromFileDescriptorRead, "Number of reads (read/pread) from a file descriptor. Does not include sockets.") \ diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1ce41204718..87b5da991a9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -51,9 +51,6 @@ namespace ProfileEvents extern const Event FailedQuery; extern const Event FailedInsertQuery; extern const Event FailedSelectQuery; - extern const Event QueryTimeMicroseconds; - extern const Event SelectQueryTimeMicroseconds; - extern const Event InsertQueryTimeMicroseconds; } namespace DB @@ -483,34 +480,8 @@ static std::tuple executeQueryImpl( query_log->add(elem); } - /// Coomon code for finish and exception callbacks - auto status_info_to_query_log = [ast](QueryLogElement &element, const QueryStatusInfo &info) mutable - { - DB::UInt64 query_time = info.elapsed_seconds * 1000000; - ProfileEvents::increment(ProfileEvents::QueryTimeMicroseconds, query_time); - if (ast->as() || ast->as()) - { - ProfileEvents::increment(ProfileEvents::SelectQueryTimeMicroseconds, query_time); - } - else if (ast->as()) - { - ProfileEvents::increment(ProfileEvents::InsertQueryTimeMicroseconds, query_time); - } - - element.query_duration_ms = info.elapsed_seconds * 1000; - - element.read_rows = info.read_rows; - element.read_bytes = info.read_bytes; - - element.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; - - element.thread_ids = std::move(info.thread_ids); - element.profile_counters = std::move(info.profile_counters); - }; - /// Also make possible for caller to log successful query finish and exception during execution. - auto finish_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, - status_info_to_query_log] + auto finish_callback = [elem, &context, log_queries, log_queries_min_type = settings.log_queries_min_type] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out, QueryPipeline * query_pipeline) mutable { QueryStatus * process_list_elem = context.getProcessListElement(); @@ -528,14 +499,21 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_FINISH; elem.event_time = time(nullptr); + elem.query_duration_ms = elapsed_seconds * 1000; - status_info_to_query_log(elem, info); + elem.read_rows = info.read_rows; + elem.read_bytes = info.read_bytes; + + elem.written_rows = info.written_rows; + elem.written_bytes = info.written_bytes; auto progress_callback = context.getProgressCallback(); if (progress_callback) progress_callback(Progress(WriteProgress(info.written_rows, info.written_bytes))); + elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; + if (stream_in) { const BlockStreamProfileInfo & stream_in_info = stream_in->getProfileInfo(); @@ -580,8 +558,7 @@ static std::tuple executeQueryImpl( } }; - auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota), - status_info_to_query_log] () mutable + auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota)] () mutable { if (quota) quota->used(Quota::ERRORS, 1, /* check_exceeded = */ false); @@ -602,7 +579,16 @@ static std::tuple executeQueryImpl( if (process_list_elem) { QueryStatusInfo info = process_list_elem->getInfo(true, current_settings.log_profile_events, false); - status_info_to_query_log(elem, info); + + elem.query_duration_ms = info.elapsed_seconds * 1000; + + elem.read_rows = info.read_rows; + elem.read_bytes = info.read_bytes; + + elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; + + elem.thread_ids = std::move(info.thread_ids); + elem.profile_counters = std::move(info.profile_counters); } if (current_settings.calculate_text_stack_trace) diff --git a/tests/queries/0_stateless/01417_query_time_in_system_events.reference b/tests/queries/0_stateless/01417_query_time_in_system_events.reference deleted file mode 100644 index 14ec14dbf6d..00000000000 --- a/tests/queries/0_stateless/01417_query_time_in_system_events.reference +++ /dev/null @@ -1,3 +0,0 @@ -QueryTimeMicroseconds: Ok -SelectQueryTimeMicroseconds: Ok -InsertQueryTimeMicroseconds: Ok diff --git a/tests/queries/0_stateless/01417_query_time_in_system_events.sh b/tests/queries/0_stateless/01417_query_time_in_system_events.sh deleted file mode 100755 index ff6d11befb0..00000000000 --- a/tests/queries/0_stateless/01417_query_time_in_system_events.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -DATA_BEFORE=`${CLICKHOUSE_CLIENT} --query="SELECT event,value FROM system.events WHERE event IN ('QueryTimeMicroseconds','SelectQueryTimeMicroseconds','InsertQueryTimeMicroseconds') FORMAT CSV"` - -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test (k UInt32) ENGINE=MergeTree ORDER BY k" -${CLICKHOUSE_CLIENT} --query="INSERT INTO test (k) SELECT sleep(1)" -${CLICKHOUSE_CLIENT} --query="SELECT sleep(1)" > /dev/null -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test" - -DATA_AFTER=`${CLICKHOUSE_CLIENT} --query="SELECT event,value FROM system.events WHERE event IN ('QueryTimeMicroseconds','SelectQueryTimeMicroseconds','InsertQueryTimeMicroseconds') FORMAT CSV"` - -declare -A VALUES_BEFORE -VALUES_BEFORE=(["\"QueryTimeMicroseconds\""]="0" ["\"SelectQueryTimeMicroseconds\""]="0" ["\"InsertQueryTimeMicroseconds\""]="0") -declare -A VALUES_AFTER -VALUES_AFTER=(["\"QueryTimeMicroseconds\""]="0" ["\"SelectQueryTimeMicroseconds\""]="0" ["\"InsertQueryTimeMicroseconds\""]="0") - -for RES in ${DATA_BEFORE} -do - IFS=',' read -ra FIELDS <<< ${RES} - VALUES_BEFORE[${FIELDS[0]}]=${FIELDS[1]} -done - -for RES in ${DATA_AFTER} -do - IFS=',' read -ra FIELDS <<< ${RES} - VALUES_AFTER[${FIELDS[0]}]=${FIELDS[1]} -done - -let QUERY_TIME=${VALUES_AFTER[\"QueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"QueryTimeMicroseconds\"]} -let SELECT_QUERY_TIME=${VALUES_AFTER[\"SelectQueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"SelectQueryTimeMicroseconds\"]} -let INSERT_QUERY_TIME=${VALUES_AFTER[\"InsertQueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"InsertQueryTimeMicroseconds\"]} -if [[ "${QUERY_TIME}" -lt "2000000" ]]; then - echo "QueryTimeMicroseconds: Fail (${QUERY_TIME})" -else - echo "QueryTimeMicroseconds: Ok" -fi -if [[ "${SELECT_QUERY_TIME}" -lt "1000000" ]]; then - echo "SelectQueryTimeMicroseconds: Fail (${SELECT_QUERY_TIME})" -else - echo "SelectQueryTimeMicroseconds: Ok" -fi -if [[ "${INSERT_QUERY_TIME}" -lt "1000000" ]]; then - echo "InsertQueryTimeMicroseconds: Fail (${INSERT_QUERY_TIME})" -else - echo "InsertQueryTimeMicroseconds: Ok" -fi - From d4266d9619a346487db55d67a07368996c80259b Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 4 Aug 2020 02:11:39 +0300 Subject: [PATCH 10/10] fix left asof join with join_use_nulls (#13291) --- src/Interpreters/HashJoin.cpp | 102 +++++++++-------- src/Interpreters/HashJoin.h | 8 +- src/Interpreters/RowRefs.cpp | 24 ++-- src/Interpreters/RowRefs.h | 6 +- .../01428_nullable_asof_join.reference | 20 ++++ .../0_stateless/01428_nullable_asof_join.sql | 105 ++++++++++++++++++ 6 files changed, 199 insertions(+), 66 deletions(-) create mode 100644 tests/queries/0_stateless/01428_nullable_asof_join.reference create mode 100644 tests/queries/0_stateless/01428_nullable_asof_join.sql diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 27294a57675..a807a9fa4ee 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -33,7 +33,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_TYPE_OF_FIELD; extern const int NOT_IMPLEMENTED; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int INCOMPATIBLE_TYPE_OF_JOIN; @@ -158,25 +157,21 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s } else if (strictness == ASTTableJoin::Strictness::Asof) { - if (kind != ASTTableJoin::Kind::Left and kind != ASTTableJoin::Kind::Inner) - throw Exception("ASOF only supports LEFT and INNER as base joins", ErrorCodes::NOT_IMPLEMENTED); + /// @note ASOF JOIN is not INNER. It's better avoid use of 'INNER ASOF' combination in messages. + /// In fact INNER means 'LEFT SEMI ASOF' while LEFT means 'LEFT OUTER ASOF'. + if (!isLeft(kind) && !isInner(kind)) + throw Exception("Wrong ASOF JOIN type. Only ASOF and LEFT ASOF joins are supported", ErrorCodes::NOT_IMPLEMENTED); + + if (key_columns.size() <= 1) + throw Exception("ASOF join needs at least one equi-join column", ErrorCodes::SYNTAX_ERROR); + + if (right_table_keys.getByName(key_names_right.back()).type->isNullable()) + throw Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED); - const IColumn * asof_column = key_columns.back(); size_t asof_size; - - asof_type = AsofRowRefs::getTypeSize(asof_column, asof_size); - if (!asof_type) - { - std::string msg = "ASOF join not supported for type: "; - msg += asof_column->getFamilyName(); - throw Exception(msg, ErrorCodes::BAD_TYPE_OF_FIELD); - } - + asof_type = AsofRowRefs::getTypeSize(*key_columns.back(), asof_size); key_columns.pop_back(); - if (key_columns.empty()) - throw Exception("ASOF join cannot be done without a joining column", ErrorCodes::SYNTAX_ERROR); - /// this is going to set up the appropriate hash table for the direct lookup part of the join /// However, this does not depend on the size of the asof join key (as that goes into the BST) /// Therefore, add it back in such that it can be extracted appropriately from the full stored @@ -248,11 +243,6 @@ HashJoin::Type HashJoin::chooseMethod(const ColumnRawPtrs & key_columns, Sizes & return Type::hashed; } -static const IColumn * extractAsofColumn(const ColumnRawPtrs & key_columns) -{ - return key_columns.back(); -} - template static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes & key_sizes) { @@ -428,14 +418,15 @@ namespace } static ALWAYS_INLINE void insertAsof(HashJoin & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool, - const IColumn * asof_column) + const IColumn & asof_column) { auto emplace_result = key_getter.emplaceKey(map, i, pool); typename Map::mapped_type * time_series_map = &emplace_result.getMapped(); + TypeIndex asof_type = *join.getAsofType(); if (emplace_result.isInserted()) - time_series_map = new (time_series_map) typename Map::mapped_type(join.getAsofType()); - time_series_map->insert(join.getAsofType(), asof_column, stored_block, i); + time_series_map = new (time_series_map) typename Map::mapped_type(asof_type); + time_series_map->insert(asof_type, asof_column, stored_block, i); } }; @@ -451,7 +442,7 @@ namespace const IColumn * asof_column [[maybe_unused]] = nullptr; if constexpr (is_asof_join) - asof_column = extractAsofColumn(key_columns); + asof_column = key_columns.back(); auto key_getter = createKeyGetter(key_columns, key_sizes); @@ -461,7 +452,7 @@ namespace continue; if constexpr (is_asof_join) - Inserter::insertAsof(join, map, key_getter, stored_block, i, pool, asof_column); + Inserter::insertAsof(join, map, key_getter, stored_block, i, pool, *asof_column); else if constexpr (mapped_one) Inserter::insertOne(join, map, key_getter, stored_block, i, pool); else @@ -614,21 +605,22 @@ class AddedColumns public: using TypeAndNames = std::vector>; - AddedColumns(const Block & sample_block_with_columns_to_add, - const Block & block_with_columns_to_add, + AddedColumns(const Block & block_with_columns_to_add, const Block & block, const Block & saved_block_sample, - const ColumnsWithTypeAndName & extras, - const HashJoin & join_, + const HashJoin & join, const ColumnRawPtrs & key_columns_, - const Sizes & key_sizes_) - : join(join_) - , key_columns(key_columns_) + const Sizes & key_sizes_, + bool is_asof_join) + : key_columns(key_columns_) , key_sizes(key_sizes_) , rows_to_add(block.rows()) - , need_filter(false) + , asof_type(join.getAsofType()) + , asof_inequality(join.getAsofInequality()) { - size_t num_columns_to_add = sample_block_with_columns_to_add.columns(); + size_t num_columns_to_add = block_with_columns_to_add.columns(); + if (is_asof_join) + ++num_columns_to_add; columns.reserve(num_columns_to_add); type_name.reserve(num_columns_to_add); @@ -641,8 +633,12 @@ public: addColumn(src_column); } - for (const auto & extra : extras) - addColumn(extra); + if (is_asof_join) + { + const ColumnWithTypeAndName & right_asof_column = join.rightAsofKeyColumn(); + addColumn(right_asof_column); + left_asof_key = key_columns.back(); + } for (auto & tn : type_name) right_indexes.push_back(saved_block_sample.getPositionByName(tn.second)); @@ -680,18 +676,25 @@ public: } } - const HashJoin & join; + TypeIndex asofType() const { return *asof_type; } + ASOF::Inequality asofInequality() const { return asof_inequality; } + const IColumn & leftAsofKey() const { return *left_asof_key; } + const ColumnRawPtrs & key_columns; const Sizes & key_sizes; size_t rows_to_add; std::unique_ptr offsets_to_replicate; - bool need_filter; + bool need_filter = false; private: TypeAndNames type_name; MutableColumns columns; std::vector right_indexes; size_t lazy_defaults_count = 0; + /// for ASOF + std::optional asof_type; + ASOF::Inequality asof_inequality; + const IColumn * left_asof_key = nullptr; void addColumn(const ColumnWithTypeAndName & src_column) { @@ -760,10 +763,6 @@ NO_INLINE IColumn::Filter joinRightColumns(const Map & map, AddedColumns & added if constexpr (need_replication) added_columns.offsets_to_replicate = std::make_unique(rows); - const IColumn * asof_column [[maybe_unused]] = nullptr; - if constexpr (is_asof_join) - asof_column = extractAsofColumn(added_columns.key_columns); - auto key_getter = createKeyGetter(added_columns.key_columns, added_columns.key_sizes); IColumn::Offset current_offset = 0; @@ -790,8 +789,11 @@ NO_INLINE IColumn::Filter joinRightColumns(const Map & map, AddedColumns & added if constexpr (is_asof_join) { - const HashJoin & join = added_columns.join; - if (const RowRef * found = mapped.findAsof(join.getAsofType(), join.getAsofInequality(), asof_column, i)) + TypeIndex asof_type = added_columns.asofType(); + ASOF::Inequality asof_inequality = added_columns.asofInequality(); + const IColumn & left_asof_key = added_columns.leftAsofKey(); + + if (const RowRef * found = mapped.findAsof(asof_type, asof_inequality, left_asof_key, i)) { setUsed(filter, i); mapped.setUsed(); @@ -932,11 +934,11 @@ void HashJoin::joinBlockImpl( /// Rare case, when keys are constant or low cardinality. To avoid code bloat, simply materialize them. Columns materialized_keys = JoinCommon::materializeColumns(block, key_names_left); - ColumnRawPtrs key_columns = JoinCommon::getRawPointers(materialized_keys); + ColumnRawPtrs left_key_columns = JoinCommon::getRawPointers(materialized_keys); /// Keys with NULL value in any column won't join to anything. ConstNullMapPtr null_map{}; - ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); + ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(left_key_columns, null_map); size_t existing_columns = block.columns(); @@ -957,12 +959,8 @@ void HashJoin::joinBlockImpl( * but they will not be used at this stage of joining (and will be in `AdderNonJoined`), and they need to be skipped. * For ASOF, the last column is used as the ASOF column */ - ColumnsWithTypeAndName extras; - if constexpr (is_asof_join) - extras.push_back(right_table_keys.getByName(key_names_right.back())); - AddedColumns added_columns(sample_block_with_columns_to_add, block_with_columns_to_add, block, savedBlockSample(), - extras, *this, key_columns, key_sizes); + AddedColumns added_columns(block_with_columns_to_add, block, savedBlockSample(), *this, left_key_columns, key_sizes, is_asof_join); bool has_required_right_keys = (required_right_keys.columns() != 0); added_columns.need_filter = need_filter || has_required_right_keys; diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 67d83d27a6d..fb879e2c507 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -191,10 +191,16 @@ public: ASTTableJoin::Kind getKind() const { return kind; } ASTTableJoin::Strictness getStrictness() const { return strictness; } - TypeIndex getAsofType() const { return *asof_type; } + const std::optional & getAsofType() const { return asof_type; } ASOF::Inequality getAsofInequality() const { return asof_inequality; } bool anyTakeLastRow() const { return any_take_last_row; } + const ColumnWithTypeAndName & rightAsofKeyColumn() const + { + /// It should be nullable if nullable_right_side is true + return savedBlockSample().getByName(key_names_right.back()); + } + /// Different types of keys for maps. #define APPLY_FOR_JOIN_VARIANTS(M) \ M(key8) \ diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index 879a0bcf88e..a206456f1b6 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; +} + namespace { @@ -56,7 +61,7 @@ AsofRowRefs::AsofRowRefs(TypeIndex type) callWithType(type, call); } -void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Block * block, size_t row_num) +void AsofRowRefs::insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num) { auto call = [&](const auto & t) { @@ -64,9 +69,9 @@ void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Bloc using LookupPtr = typename Entry::LookupPtr; using ColumnType = std::conditional_t, ColumnDecimal, ColumnVector>; - auto * column = typeid_cast(asof_column); + const auto & column = typeid_cast(asof_column); - T key = column->getElement(row_num); + T key = column.getElement(row_num); auto entry = Entry(key, RowRef(block, row_num)); std::get(lookups)->insert(entry); }; @@ -74,7 +79,7 @@ void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Bloc callWithType(type, call); } -const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn * asof_column, size_t row_num) const +const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const { const RowRef * out = nullptr; @@ -88,8 +93,8 @@ const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality using LookupPtr = typename EntryType::LookupPtr; using ColumnType = std::conditional_t, ColumnDecimal, ColumnVector>; - auto * column = typeid_cast(asof_column); - T key = column->getElement(row_num); + const auto & column = typeid_cast(asof_column); + T key = column.getElement(row_num); auto & typed_lookup = std::get(lookups); if (is_strict) @@ -102,9 +107,9 @@ const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality return out; } -std::optional AsofRowRefs::getTypeSize(const IColumn * asof_column, size_t & size) +std::optional AsofRowRefs::getTypeSize(const IColumn & asof_column, size_t & size) { - TypeIndex idx = asof_column->getDataType(); + TypeIndex idx = asof_column.getDataType(); switch (idx) { @@ -152,8 +157,7 @@ std::optional AsofRowRefs::getTypeSize(const IColumn * asof_column, s break; } - size = 0; - return {}; + throw Exception("ASOF join not supported for type: " + std::string(asof_column.getFamilyName()), ErrorCodes::BAD_TYPE_OF_FIELD); } } diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index e8231b1c233..fc035bf626e 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -233,13 +233,13 @@ public: AsofRowRefs() {} AsofRowRefs(TypeIndex t); - static std::optional getTypeSize(const IColumn * asof_column, size_t & type_size); + static std::optional getTypeSize(const IColumn & asof_column, size_t & type_size); // This will be synchronized by the rwlock mutex in Join.h - void insert(TypeIndex type, const IColumn * asof_column, const Block * block, size_t row_num); + void insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num); // This will internally synchronize - const RowRef * findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn * asof_column, size_t row_num) const; + const RowRef * findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const; private: // Lookups can be stored in a HashTable because it is memmovable diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.reference b/tests/queries/0_stateless/01428_nullable_asof_join.reference new file mode 100644 index 00000000000..f04655fefaa --- /dev/null +++ b/tests/queries/0_stateless/01428_nullable_asof_join.reference @@ -0,0 +1,20 @@ +left asof using +0 \N 0 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +1 \N 1 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +1 1 2 2 UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +0 \N 0 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +1 \N 1 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +1 1 2 2 UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +left asof on +0 \N 0 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +1 \N 1 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +1 1 2 2 UInt8 Nullable(UInt8) UInt8 Nullable(UInt8) +0 \N 0 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +1 \N 1 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +1 1 2 2 UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8) +asof using +1 1 2 2 UInt8 UInt8 UInt8 UInt8 +1 1 2 2 UInt8 UInt8 Nullable(UInt8) UInt8 +asof on +1 1 2 2 UInt8 UInt8 UInt8 UInt8 +1 1 2 2 UInt8 UInt8 Nullable(UInt8) UInt8 diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.sql b/tests/queries/0_stateless/01428_nullable_asof_join.sql new file mode 100644 index 00000000000..c812e6ecfab --- /dev/null +++ b/tests/queries/0_stateless/01428_nullable_asof_join.sql @@ -0,0 +1,105 @@ +SET join_use_nulls = 1; + +select 'left asof using'; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b +USING(pk, dt) +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b +USING(pk, dt) +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b +USING(pk, dt) +ORDER BY a.dt;-- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b +USING(pk, dt) +ORDER BY a.dt;-- { serverError 48 } + +select 'left asof on'; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt;-- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt;-- { serverError 48 } + +select 'asof using'; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, 2 as dt) b +USING(pk, dt) +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, 2 as dt) b +USING(pk, dt) +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +USING(pk, dt) +ORDER BY a.dt;-- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +USING(pk, dt) +ORDER BY a.dt;-- { serverError 48 } + +select 'asof on'; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, 2 as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, 2 as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt; + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt;-- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.pk = b.pk AND a.dt >= b.dt +ORDER BY a.dt;-- { serverError 48 }