From 203cafa9b6e421971f3d2d67051929d40d3a21f9 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Sun, 7 Feb 2021 14:32:19 +0000 Subject: [PATCH 001/464] Mark functions as pure virtual to fix missing vtable pointer error ``` Undefined symbols for architecture x86_64: "vtable for DB::IModel", referenced from: DB::IModel::IModel() in Obfuscator.cpp.o NOTE: a missing vtable usually means the first non-inline virtual member function has no definition. ld: symbol(s) not found for architecture x86_64 clang-11: error: linker command failed with exit code 1 (use -v to see invocation) ninja: build stopped: subcommand failed. ``` --- programs/obfuscator/Obfuscator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 950db4e4f05..5eb5467c58b 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -100,16 +100,16 @@ class IModel { public: /// Call train iteratively for each block to train a model. - virtual void train(const IColumn & column); + virtual void train(const IColumn & column) = 0; /// Call finalize one time after training before generating. - virtual void finalize(); + virtual void finalize() = 0; /// Call generate: pass source data column to obtain a column with anonymized data as a result. - virtual ColumnPtr generate(const IColumn & column); + virtual ColumnPtr generate(const IColumn & column) = 0; /// Deterministically change seed to some other value. This can be used to generate more values than were in source. - virtual void updateSeed(); + virtual void updateSeed() = 0; virtual ~IModel() = default; }; From 479b45d772f18b690f1da7365d9a582ad836b577 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Sun, 7 Feb 2021 14:37:50 +0000 Subject: [PATCH 002/464] Fix linker flags for shared linking on macOS This combination now works: `-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1`. Without `SPLIT_SHARED_LIBRARIES` it is still failing. --- base/common/CMakeLists.txt | 4 ++++ base/daemon/CMakeLists.txt | 5 +++++ src/CMakeLists.txt | 12 ++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index cea52b443dd..217976e2fb0 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -47,6 +47,10 @@ endif() target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..) +if (OS_DARWIN AND NOT MAKE_STATIC_LIBRARIES) + target_link_libraries(common PUBLIC -Wl,-U,_inside_main) +endif() + # Allow explicit fallback to readline if (NOT ENABLE_REPLXX AND ENABLE_READLINE) message (STATUS "Attempt to fallback to readline explicitly") diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 26d59a57e7f..6ef87db6a61 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -5,6 +5,11 @@ add_library (daemon ) target_include_directories (daemon PUBLIC ..) + +if (OS_DARWIN AND NOT MAKE_STATIC_LIBRARIES) + target_link_libraries (daemon PUBLIC -Wl,-undefined,dynamic_lookup) +endif() + target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) if (USE_SENTRY) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dba9385fe27..8acd8b32c39 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -157,7 +157,11 @@ macro(add_object_library name common_path) list (APPEND all_modules ${name}) add_headers_and_sources(${name} ${common_path}) add_library(${name} SHARED ${${name}_sources} ${${name}_headers}) - target_link_libraries (${name} PRIVATE -Wl,--unresolved-symbols=ignore-all) + if (OS_DARWIN) + target_link_libraries (${name} PRIVATE -Wl,-undefined,dynamic_lookup) + else() + target_link_libraries (${name} PRIVATE -Wl,--unresolved-symbols=ignore-all) + endif() endif () endmacro() @@ -209,7 +213,11 @@ else() target_link_libraries (clickhouse_interpreters PRIVATE clickhouse_parsers_new jemalloc libdivide) list (APPEND all_modules dbms) # force all split libs to be linked - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed") + if (OS_DARWIN) + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-undefined,error") + else() + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed") + endif() endif () macro (dbms_target_include_directories) From 813092ff55c9534a0d10be577a3fbb8326810442 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 12:49:38 +0000 Subject: [PATCH 003/464] Test replicated fetches_network partition --- .../__init__.py | 0 .../configs/profiles.xml | 9 ++ .../test.py | 82 +++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 tests/integration/test_replicated_fetches_network_partition/__init__.py create mode 100644 tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml create mode 100644 tests/integration/test_replicated_fetches_network_partition/test.py diff --git a/tests/integration/test_replicated_fetches_network_partition/__init__.py b/tests/integration/test_replicated_fetches_network_partition/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml b/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml new file mode 100644 index 00000000000..7abcf0bfde3 --- /dev/null +++ b/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml @@ -0,0 +1,9 @@ + + + + + 30 + + + + \ No newline at end of file diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py new file mode 100644 index 00000000000..618bf94c2bd --- /dev/null +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import pytest +import time +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +import random +import string + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, user_configs=['configs/profiles.xml']) + +DEFAULT_MAX_THREADS_FOR_FETCH = 3 + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_random_string(length): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + + +def test_no_stall(started_cluster): + node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple() PARTITION BY key") + node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple() PARTITION BY key") + node3.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '3') ORDER BY tuple() PARTITION BY key") + + node1.query("SYSTEM STOP MERGES") + node2.query("SYSTEM STOP MERGES") + node3.query("SYSTEM STOP MERGES") + + with PartitionManager() as pm: + node3.query("SYSTEM STOP FETCHES t") + + node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format(get_random_string(104857))) + + # Make sure node2 has all the parts. + node2.query("SYSTEM SYNC REPLICA t") + + # Do not allow sending from replica 2 yet, force node3 to initiate replication from node1. + node2.query("SYSTEM STOP REPLICATED SENDS") + + print("replica 2 fully synced") + + # Make node1 very slow, node3 should replicate from node2 instead. + pm.add_network_delay(node1, 1000) + + # node3 starts to replicate from node 1 + node3.query("SYSTEM START FETCHES t") + + # Wait some time to give a chance for node3 to try replicating without success from node1. + time.sleep(10) + + # Wait for replication... + node2.query("SYSTEM START REPLICATED SENDS") + + for _ in range(1000): + print(node3.query("SELECT result_part_name FROM system.replicated_fetches").strip().split()) + print() + result = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() + print(result) + print() + print() + + # Replication done. + if result == "5": + break + + time.sleep(3) From b153e8c190bc6246456cdc401beb8254004897cd Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 17:25:10 +0000 Subject: [PATCH 004/464] Add support for custom fetchPart timeouts --- src/Storages/MergeTree/MergeTreeSettings.h | 3 +++ src/Storages/StorageReplicatedMergeTree.cpp | 23 +++++++++++++++++-- src/Storages/StorageReplicatedMergeTree.h | 2 ++ .../test.py | 11 ++++----- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 53388617a07..705222e86f5 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -80,6 +80,9 @@ struct Settings; M(UInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT, "Limit parallel fetches from endpoint (actually pool size).", 0) \ M(UInt64, replicated_max_parallel_sends, 0, "Limit parallel sends.", 0) \ M(UInt64, replicated_max_parallel_sends_for_table, 0, "Limit parallel sends for one table.", 0) \ + M(Seconds, replicated_fetches_http_connection_timeout, 0, "HTTP connection timeout for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_send_timeout, 0, "HTTP send timeout for part fetch/send requests. Inherited from default profile `http_send_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_receive_timeout, 0, "HTTP receive timeout for fetch/send part requests. Inherited from default profile `http_receive_timeout` if not set explicitly.", 0) \ M(Bool, replicated_can_become_leader, true, "If true, Replicated tables replicas on this node will try to acquire leadership.", 0) \ M(Seconds, zookeeper_session_expiration_check_period, 60, "ZooKeeper session expiration check period, in seconds.", 0) \ M(Bool, detach_old_local_parts_when_cloning_replica, 1, "Do not remove old local parts when repairing lost replica.", 0) \ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 69cbe0d7062..6b168f051ac 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2178,7 +2178,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) { String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + auto timeouts = getFetchPartHTTPTimeouts(global_context); + auto [user, password] = global_context.getInterserverCredentials(); String interserver_scheme = global_context.getInterserverScheme(); @@ -3111,6 +3112,23 @@ void StorageReplicatedMergeTree::exitLeaderElection() leader_election = nullptr; } +ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(const Context & context) +{ + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context); + auto settings = getSettings(); + + if (settings->replicated_fetches_http_connection_timeout.changed) + timeouts.connection_timeout = settings->replicated_fetches_http_connection_timeout; + + if (settings->replicated_fetches_http_send_timeout.changed) + timeouts.send_timeout = settings->replicated_fetches_http_send_timeout; + + if (settings->replicated_fetches_http_receive_timeout.changed) + timeouts.receive_timeout = settings->replicated_fetches_http_receive_timeout; + + return timeouts; +} + bool StorageReplicatedMergeTree::checkReplicaHavePart(const String & replica, const String & part_name) { auto zookeeper = getZooKeeper(); @@ -3520,7 +3538,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora else { ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + auto timeouts = getFetchPartHTTPTimeouts(global_context); + auto user_password = global_context.getInterserverCredentials(); String interserver_scheme = global_context.getInterserverScheme(); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 6db05294b63..2acfd7027b9 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -488,6 +488,8 @@ private: /// Exchange parts. + ConnectionTimeouts getFetchPartHTTPTimeouts(const Context & context); + /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py index 618bf94c2bd..e42395c7af4 100644 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -68,15 +68,12 @@ def test_no_stall(started_cluster): node2.query("SYSTEM START REPLICATED SENDS") for _ in range(1000): - print(node3.query("SELECT result_part_name FROM system.replicated_fetches").strip().split()) - print() - result = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() - print(result) - print() - print() + print('Currently running fetches', node3.query("SELECT result_part_name FROM system.replicated_fetches").strip().split()) + parts_fetched = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() + print('parts_fetched', parts_fetched) # Replication done. - if result == "5": + if parts_fetched == "5": break time.sleep(3) From 75fca08b2a15c702884aa760fbca3389c974bea3 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 17:26:53 +0000 Subject: [PATCH 005/464] Update test_replicated_fetches_network_partition with new settings --- .../configs/merge_tree.xml | 6 ++++++ .../configs/profiles.xml | 9 --------- .../test_replicated_fetches_network_partition/test.py | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml delete mode 100644 tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml diff --git a/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml b/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml new file mode 100644 index 00000000000..eba2c5e8ffc --- /dev/null +++ b/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml @@ -0,0 +1,6 @@ + + + 30 + 1 + + diff --git a/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml b/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml deleted file mode 100644 index 7abcf0bfde3..00000000000 --- a/tests/integration/test_replicated_fetches_network_partition/configs/profiles.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - 30 - - - - \ No newline at end of file diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py index e42395c7af4..e493709dcbe 100644 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -10,7 +10,7 @@ import string cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, user_configs=['configs/profiles.xml']) +node3 = cluster.add_instance('node3', with_zookeeper=True, main_configs=['configs/merge_tree.xml']) DEFAULT_MAX_THREADS_FOR_FETCH = 3 From 165ba59a1535cf97a7bc35a5dfbf37904163dd5b Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 20:05:22 +0000 Subject: [PATCH 006/464] Increase network delay and add more info to logs for debugging CI behaviour is different from my test environment --- .../test_replicated_fetches_network_partition/test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py index e493709dcbe..fd4727ba9fb 100644 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -12,8 +12,6 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) node3 = cluster.add_instance('node3', with_zookeeper=True, main_configs=['configs/merge_tree.xml']) -DEFAULT_MAX_THREADS_FOR_FETCH = 3 - @pytest.fixture(scope="module") def started_cluster(): try: @@ -56,7 +54,7 @@ def test_no_stall(started_cluster): print("replica 2 fully synced") # Make node1 very slow, node3 should replicate from node2 instead. - pm.add_network_delay(node1, 1000) + pm.add_network_delay(node1, 2000) # node3 starts to replicate from node 1 node3.query("SYSTEM START FETCHES t") @@ -68,7 +66,7 @@ def test_no_stall(started_cluster): node2.query("SYSTEM START REPLICATED SENDS") for _ in range(1000): - print('Currently running fetches', node3.query("SELECT result_part_name FROM system.replicated_fetches").strip().split()) + print('Currently running fetches', node3.query("SELECT result_part_name, source_replica_hostname, progress FROM system.replicated_fetches").strip().split()) parts_fetched = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() print('parts_fetched', parts_fetched) From f39eef92279c53edf00eefad7a328f2d98a8eb73 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 20:21:20 +0000 Subject: [PATCH 007/464] Reformat test and drop tables at the end to reduce the size of test results archive --- .../test.py | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py index fd4727ba9fb..e0c5c4cce8a 100644 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -12,6 +12,7 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) node3 = cluster.add_instance('node3', with_zookeeper=True, main_configs=['configs/merge_tree.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -36,23 +37,24 @@ def test_no_stall(started_cluster): node2.query("SYSTEM STOP MERGES") node3.query("SYSTEM STOP MERGES") + # Pause node3 until the test setup is prepared + node3.query("SYSTEM STOP FETCHES t") + + node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format(get_random_string(104857))) + + # Make sure node2 has all the parts. + node2.query("SYSTEM SYNC REPLICA t") + + # Do not allow sending from replica 2 yet, force node3 to initiate replication from node1. + node2.query("SYSTEM STOP REPLICATED SENDS") + + print("replica 2 fully synced") + with PartitionManager() as pm: - node3.query("SYSTEM STOP FETCHES t") - - node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format(get_random_string(104857))) - - # Make sure node2 has all the parts. - node2.query("SYSTEM SYNC REPLICA t") - - # Do not allow sending from replica 2 yet, force node3 to initiate replication from node1. - node2.query("SYSTEM STOP REPLICATED SENDS") - - print("replica 2 fully synced") - # Make node1 very slow, node3 should replicate from node2 instead. pm.add_network_delay(node1, 2000) @@ -66,12 +68,20 @@ def test_no_stall(started_cluster): node2.query("SYSTEM START REPLICATED SENDS") for _ in range(1000): - print('Currently running fetches', node3.query("SELECT result_part_name, source_replica_hostname, progress FROM system.replicated_fetches").strip().split()) + print('Currently running fetches:') + print(node3.query("SELECT result_part_name, source_replica_hostname, progress FROM system.replicated_fetches").strip()) + print() + parts_fetched = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() - print('parts_fetched', parts_fetched) + print('parts_fetched:', parts_fetched) + print() # Replication done. if parts_fetched == "5": break time.sleep(3) + + node1.query("DROP TABLE t SYNC") + node2.query("DROP TABLE t SYNC") + node3.query("DROP TABLE t SYNC") From 135f82ce94d8cbb548de5a6f0a0712bffa0d4a9c Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 4 Feb 2021 20:33:06 +0000 Subject: [PATCH 008/464] Correctly document that settings apply only for fetch part requests --- src/Storages/MergeTree/MergeTreeSettings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 705222e86f5..4d7be425cf7 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -81,8 +81,8 @@ struct Settings; M(UInt64, replicated_max_parallel_sends, 0, "Limit parallel sends.", 0) \ M(UInt64, replicated_max_parallel_sends_for_table, 0, "Limit parallel sends for one table.", 0) \ M(Seconds, replicated_fetches_http_connection_timeout, 0, "HTTP connection timeout for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly.", 0) \ - M(Seconds, replicated_fetches_http_send_timeout, 0, "HTTP send timeout for part fetch/send requests. Inherited from default profile `http_send_timeout` if not set explicitly.", 0) \ - M(Seconds, replicated_fetches_http_receive_timeout, 0, "HTTP receive timeout for fetch/send part requests. Inherited from default profile `http_receive_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_send_timeout, 0, "HTTP send timeout for part fetch requests. Inherited from default profile `http_send_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_receive_timeout, 0, "HTTP receive timeout for fetch part requests. Inherited from default profile `http_receive_timeout` if not set explicitly.", 0) \ M(Bool, replicated_can_become_leader, true, "If true, Replicated tables replicas on this node will try to acquire leadership.", 0) \ M(Seconds, zookeeper_session_expiration_check_period, 60, "ZooKeeper session expiration check period, in seconds.", 0) \ M(Bool, detach_old_local_parts_when_cloning_replica, 1, "Do not remove old local parts when repairing lost replica.", 0) \ From 6181daf92d9aac636bc2c2df7b7a9f0944ac35bc Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Fri, 5 Feb 2021 00:35:54 +0000 Subject: [PATCH 009/464] Workaround for drop not finishing if it is started while table is readonly --- .../test_replicated_fetches_network_partition/test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py index e0c5c4cce8a..b0b5534cf1f 100644 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ b/tests/integration/test_replicated_fetches_network_partition/test.py @@ -82,6 +82,7 @@ def test_no_stall(started_cluster): time.sleep(3) - node1.query("DROP TABLE t SYNC") - node2.query("DROP TABLE t SYNC") - node3.query("DROP TABLE t SYNC") + for n in [node1, node2, node3]: + # Workaround for drop not finishing if it is started while table is readonly. + n.query("SYSTEM RESTART REPLICA t") + n.query("DROP TABLE t SYNC") From 84489b82433783f32572f8150dbd8ef4d1959acd Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 8 Feb 2021 21:19:32 +0000 Subject: [PATCH 010/464] Improve replicated fetches timeouts test and make it 3x faster --- .../configs/merge_tree.xml | 6 -- .../test.py | 88 ----------------- .../__init__.py | 0 .../configs/server.xml | 3 + .../test_replicated_fetches_timeouts/test.py | 95 +++++++++++++++++++ 5 files changed, 98 insertions(+), 94 deletions(-) delete mode 100644 tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml delete mode 100644 tests/integration/test_replicated_fetches_network_partition/test.py rename tests/integration/{test_replicated_fetches_network_partition => test_replicated_fetches_timeouts}/__init__.py (100%) create mode 100644 tests/integration/test_replicated_fetches_timeouts/configs/server.xml create mode 100644 tests/integration/test_replicated_fetches_timeouts/test.py diff --git a/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml b/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml deleted file mode 100644 index eba2c5e8ffc..00000000000 --- a/tests/integration/test_replicated_fetches_network_partition/configs/merge_tree.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - 30 - 1 - - diff --git a/tests/integration/test_replicated_fetches_network_partition/test.py b/tests/integration/test_replicated_fetches_network_partition/test.py deleted file mode 100644 index b0b5534cf1f..00000000000 --- a/tests/integration/test_replicated_fetches_network_partition/test.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - -import pytest -import time -from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -import random -import string - -cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, main_configs=['configs/merge_tree.xml']) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - - yield cluster - - finally: - cluster.shutdown() - - -def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) - - -def test_no_stall(started_cluster): - node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple() PARTITION BY key") - node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple() PARTITION BY key") - node3.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '3') ORDER BY tuple() PARTITION BY key") - - node1.query("SYSTEM STOP MERGES") - node2.query("SYSTEM STOP MERGES") - node3.query("SYSTEM STOP MERGES") - - # Pause node3 until the test setup is prepared - node3.query("SYSTEM STOP FETCHES t") - - node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format(get_random_string(104857))) - - # Make sure node2 has all the parts. - node2.query("SYSTEM SYNC REPLICA t") - - # Do not allow sending from replica 2 yet, force node3 to initiate replication from node1. - node2.query("SYSTEM STOP REPLICATED SENDS") - - print("replica 2 fully synced") - - with PartitionManager() as pm: - # Make node1 very slow, node3 should replicate from node2 instead. - pm.add_network_delay(node1, 2000) - - # node3 starts to replicate from node 1 - node3.query("SYSTEM START FETCHES t") - - # Wait some time to give a chance for node3 to try replicating without success from node1. - time.sleep(10) - - # Wait for replication... - node2.query("SYSTEM START REPLICATED SENDS") - - for _ in range(1000): - print('Currently running fetches:') - print(node3.query("SELECT result_part_name, source_replica_hostname, progress FROM system.replicated_fetches").strip()) - print() - - parts_fetched = node3.query("SELECT count() FROM system.parts WHERE table = 't'").strip() - print('parts_fetched:', parts_fetched) - print() - - # Replication done. - if parts_fetched == "5": - break - - time.sleep(3) - - for n in [node1, node2, node3]: - # Workaround for drop not finishing if it is started while table is readonly. - n.query("SYSTEM RESTART REPLICA t") - n.query("DROP TABLE t SYNC") diff --git a/tests/integration/test_replicated_fetches_network_partition/__init__.py b/tests/integration/test_replicated_fetches_timeouts/__init__.py similarity index 100% rename from tests/integration/test_replicated_fetches_network_partition/__init__.py rename to tests/integration/test_replicated_fetches_timeouts/__init__.py diff --git a/tests/integration/test_replicated_fetches_timeouts/configs/server.xml b/tests/integration/test_replicated_fetches_timeouts/configs/server.xml new file mode 100644 index 00000000000..d4b441b91fb --- /dev/null +++ b/tests/integration/test_replicated_fetches_timeouts/configs/server.xml @@ -0,0 +1,3 @@ + + 0.1 + diff --git a/tests/integration/test_replicated_fetches_timeouts/test.py b/tests/integration/test_replicated_fetches_timeouts/test.py new file mode 100644 index 00000000000..963ec2487fd --- /dev/null +++ b/tests/integration/test_replicated_fetches_timeouts/test.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + 'node1', with_zookeeper=True, + main_configs=['configs/server.xml']) + +node2 = cluster.add_instance( + 'node2', with_zookeeper=True, + main_configs=['configs/server.xml']) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_random_string(length): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + + +def test_no_stall(started_cluster): + for instance in started_cluster.instances.values(): + instance.query(""" + CREATE TABLE t (key UInt64, data String) + ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '{instance}') + ORDER BY tuple() + PARTITION BY key""") + + # Pause node3 until the test setup is prepared + node2.query("SYSTEM STOP FETCHES t") + + node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(500)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(500)".format(get_random_string(104857))) + + with PartitionManager() as pm: + pm.add_network_delay(node1, 2000) + node2.query("SYSTEM START FETCHES t") + + # Wait for timeout exceptions to confirm that timeout is triggered. + while True: + conn_timeout_exceptions = int(node2.query( + """ + SELECT count() + FROM system.replication_queue + WHERE last_exception LIKE '%connect timed out%' + """)) + + if conn_timeout_exceptions >= 2: + break + + time.sleep(0.1) + + print("Connection timeouts tested!") + + # Increase connection timeout and wait for receive timeouts. + node2.query(""" + ALTER TABLE t + MODIFY SETTING replicated_fetches_http_connection_timeout = 30, + replicated_fetches_http_receive_timeout = 1""") + + while True: + timeout_exceptions = int(node2.query( + """ + SELECT count() + FROM system.replication_queue + WHERE last_exception LIKE '%e.displayText() = Timeout%' + AND last_exception NOT LIKE '%connect timed out%' + """).strip()) + + if timeout_exceptions >= 2: + break + + time.sleep(0.1) + + for instance in started_cluster.instances.values(): + # Workaround for DROP TABLE not finishing if it is started while table is readonly. + instance.query("SYSTEM RESTART REPLICA t") + + # Cleanup data directory from test results archive. + instance.query("DROP TABLE t SYNC") From 81d324da152b8411396444b760e6332ec8a2315a Mon Sep 17 00:00:00 2001 From: Stig Bakken Date: Fri, 12 Feb 2021 15:12:14 +0800 Subject: [PATCH 011/464] MaterializeMySQL: add skipping index for _version column --- .../MySQL/InterpretersMySQLDDLQuery.cpp | 16 +++- .../MySQL/tests/gtest_create_rewritten.cpp | 82 +++++++++++-------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 7f4da0638d4..dfc126a6c24 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -411,13 +412,26 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( return column_declaration; }; - /// Add _sign and _version column. + /// Add _sign and _version columns. String sign_column_name = getUniqueColumnName(columns_name_and_type, "_sign"); String version_column_name = getUniqueColumnName(columns_name_and_type, "_version"); columns->set(columns->columns, InterpreterCreateQuery::formatColumns(columns_name_and_type)); columns->columns->children.emplace_back(create_materialized_column_declaration(sign_column_name, "Int8", UInt64(1))); columns->columns->children.emplace_back(create_materialized_column_declaration(version_column_name, "UInt64", UInt64(1))); + /// Add minmax skipping index for _version column. + auto version_index = std::make_shared(); + version_index->name = version_column_name; + auto index_expr = std::make_shared(version_column_name); + auto index_type = makeASTFunction("minmax"); + index_type->no_empty_args = true; + version_index->set(version_index->expr, index_expr); + version_index->set(version_index->type, index_type); + version_index->granularity = 1; + ASTPtr indices = std::make_shared(); + indices->children.push_back(version_index); + columns->set(columns->indices, indices); + auto storage = std::make_shared(); /// The `partition by` expression must use primary keys, otherwise the primary keys will not be merge. diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp index 0d8e57aafc5..5a82a570db0 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp @@ -28,6 +28,10 @@ static inline ASTPtr tryRewrittenCreateQuery(const String & query, const Context context, "test_database", "test_database")[0]; } +static const char MATERIALIZEMYSQL_TABLE_COLUMNS[] = ", `_sign` Int8() MATERIALIZED 1" + ", `_version` UInt64() MATERIALIZED 1" + ", INDEX _version _version TYPE minmax GRANULARITY 1"; + TEST(MySQLCreateRewritten, ColumnsDataType) { tryRegisterFunctions(); @@ -45,46 +49,46 @@ TEST(MySQLCreateRewritten, ColumnsDataType) { EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + ")", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(" + mapped_type + ")" - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(" + mapped_type + ")" + + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " NOT NULL)", context_holder.context)), "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` " + mapped_type + - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' NOT NULL)", context_holder.context)), "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` " + mapped_type + - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); if (Poco::toUpper(test_type).find("INT") != std::string::npos) { EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " UNSIGNED)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" + + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' UNSIGNED)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" + + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " NOT NULL UNSIGNED)", context_holder.context)), "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` U" + mapped_type + - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' UNSIGNED NOT NULL)", context_holder.context)), "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` U" + mapped_type + - ", `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); } } @@ -109,13 +113,15 @@ TEST(MySQLCreateRewritten, PartitionPolicy) { EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + " PRIMARY KEY)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + " NOT NULL PRIMARY KEY)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); } } @@ -138,23 +144,27 @@ TEST(MySQLCreateRewritten, OrderbyPolicy) { EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + " PRIMARY KEY, `key2` " + test_type + " UNIQUE KEY)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` Nullable(" + mapped_type + "), `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, assumeNotNull(key2))"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` Nullable(" + mapped_type + ")" + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, assumeNotNull(key2))"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + " NOT NULL PRIMARY KEY, `key2` " + test_type + " NOT NULL UNIQUE KEY)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` " + mapped_type + ", `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, key2)"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` " + mapped_type + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, key2)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + " KEY UNIQUE KEY)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` " + test_type + ", `key2` " + test_type + " UNIQUE KEY, PRIMARY KEY(`key`, `key2`))", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` " + mapped_type + ", `_sign` Int8() MATERIALIZED 1, " - "`_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, key2)"); + "CREATE TABLE test_database.test_table_1 (`key` " + mapped_type + ", `key2` " + mapped_type + + MATERIALIZEMYSQL_TABLE_COLUMNS + + ") ENGINE = ReplacingMergeTree(_version)" + partition_policy + " ORDER BY (key, key2)"); } } @@ -165,23 +175,27 @@ TEST(MySQLCreateRewritten, RewrittenQueryWithPrimaryKey) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` int NOT NULL PRIMARY KEY) ENGINE=InnoDB DEFAULT CHARSET=utf8", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version) " - "PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); + "CREATE TABLE test_database.test_table_1 (`key` Int32" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` int NOT NULL, PRIMARY KEY (`key`)) ENGINE=InnoDB DEFAULT CHARSET=utf8", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = ReplacingMergeTree(_version) " - "PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); + "CREATE TABLE test_database.test_table_1 (`key` Int32" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key_1` int NOT NULL, key_2 INT NOT NULL, PRIMARY KEY (`key_1`, `key_2`)) ENGINE=InnoDB DEFAULT CHARSET=utf8", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key_1` Int32, `key_2` Int32, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " - "ReplacingMergeTree(_version) PARTITION BY intDiv(key_1, 4294967) ORDER BY (key_1, key_2)"); + "CREATE TABLE test_database.test_table_1 (`key_1` Int32, `key_2` Int32" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key_1, 4294967) ORDER BY (key_1, key_2)"); EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key_1` BIGINT NOT NULL, key_2 INT NOT NULL, PRIMARY KEY (`key_1`, `key_2`)) ENGINE=InnoDB DEFAULT CHARSET=utf8", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key_1` Int64, `key_2` Int32, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " - "ReplacingMergeTree(_version) PARTITION BY intDiv(key_2, 4294967) ORDER BY (key_1, key_2)"); + "CREATE TABLE test_database.test_table_1 (`key_1` Int64, `key_2` Int32" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key_2, 4294967) ORDER BY (key_1, key_2)"); } TEST(MySQLCreateRewritten, RewrittenQueryWithPrefixKey) @@ -191,7 +205,8 @@ TEST(MySQLCreateRewritten, RewrittenQueryWithPrefixKey) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1` (`key` int NOT NULL PRIMARY KEY, `prefix_key` varchar(200) NOT NULL, KEY prefix_key_index(prefix_key(2))) ENGINE=InnoDB DEFAULT CHARSET=utf8", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `prefix_key` String, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1) ENGINE = " + "CREATE TABLE test_database.test_table_1 (`key` Int32, `prefix_key` String" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY (key, prefix_key)"); } @@ -204,6 +219,7 @@ TEST(MySQLCreateRewritten, UniqueKeysConvert) "CREATE TABLE `test_database`.`test_table_1` (code varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,name varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL," " id bigint NOT NULL AUTO_INCREMENT, tenant_id bigint NOT NULL, PRIMARY KEY (id), UNIQUE KEY code_id (code, tenant_id), UNIQUE KEY name_id (name, tenant_id))" " ENGINE=InnoDB AUTO_INCREMENT=100 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`code` String, `name` String, `id` Int64, `tenant_id` Int64, `_sign` Int8() MATERIALIZED 1, `_version` UInt64() MATERIALIZED 1)" - " ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(id, 18446744073709551) ORDER BY (code, name, tenant_id, id)"); + "CREATE TABLE test_database.test_table_1 (`code` String, `name` String, `id` Int64, `tenant_id` Int64" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(id, 18446744073709551) ORDER BY (code, name, tenant_id, id)"); } From 7fa72f238acf34b438a3e6365e2220a35b699bd6 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Mon, 22 Feb 2021 12:49:49 +0300 Subject: [PATCH 012/464] draft --- docs/en/sql-reference/functions/string-search-functions.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 83b0edea438..471768da414 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -12,7 +12,10 @@ The search is case-sensitive by default in all these functions. There are separa ## position(haystack, needle), locate(haystack, needle) {#position} -Returns the position (in bytes) of the found substring in the string, starting from 1. +Returns the position (in bytes) of the found substring in the string, starting from 1. + +Function position now supports position(needle in haystack) synax for SQL compatibility. +Enhanced function position to support SQL compatibility syntax: position(needle in haystack), which is same as position(haystack, needle). For a case-insensitive search, use the function [positionCaseInsensitive](#positioncaseinsensitive). From 08126030d69cb1771f7a6cb9a96962d347f89705 Mon Sep 17 00:00:00 2001 From: pingyu Date: Fri, 26 Feb 2021 00:44:01 +0800 Subject: [PATCH 013/464] wip #14893 --- .gitmodules | 3 +++ CMakeLists.txt | 1 + cmake/find/datasketches.cmake | 29 +++++++++++++++++++++++++++++ contrib/datasketches-cpp | 1 + src/CMakeLists.txt | 4 ++++ 5 files changed, 38 insertions(+) create mode 100644 cmake/find/datasketches.cmake create mode 160000 contrib/datasketches-cpp diff --git a/.gitmodules b/.gitmodules index 7a2c5600e65..b8e87e06758 100644 --- a/.gitmodules +++ b/.gitmodules @@ -221,3 +221,6 @@ [submodule "contrib/NuRaft"] path = contrib/NuRaft url = https://github.com/ClickHouse-Extras/NuRaft.git +[submodule "contrib/datasketches-cpp"] + path = contrib/datasketches-cpp + url = https://github.com/apache/datasketches-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 9002f1df140..b46b6e6afdd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -504,6 +504,7 @@ include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/stats.cmake) +include (cmake/find/datasketches.cmake) set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "") find_contrib_lib(cityhash) diff --git a/cmake/find/datasketches.cmake b/cmake/find/datasketches.cmake new file mode 100644 index 00000000000..93ad9e7ed48 --- /dev/null +++ b/cmake/find/datasketches.cmake @@ -0,0 +1,29 @@ +option (ENABLE_DATASKETCHES "Enable DataSketches" ${ENABLE_LIBRARIES}) + +if (ENABLE_DATASKETCHES) + +option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketches library instead of bundled" ${NOT_UNBUNDLED}) + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/CMakeLists.txt") + if (USE_INTERNAL_DATASKETCHES_LIBRARY) + message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_DATASKETCHES_LIBRARY 1) + set(USE_INTERNAL_DATASKETCHES_LIBRARY 0) +endif() + +if (USE_INTERNAL_DATASKETCHES_LIBRARY) + set(DATASKETCHES_LIBRARY theta) + set(DATASKETCHES_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/common/include" "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/include") +elseif (NOT MISSING_INTERNAL_DATASKETCHES_LIBRARY) + find_library(DATASKETCHES_LIBRARY theta) + find_path(DATASKETCHES_INCLUDE_DIR NAMES theta_sketch.hpp PATHS ${DATASKETCHES_INCLUDE_PATHS}) +endif() + +if (DATASKETCHES_LIBRARY AND DATASKETCHES_INCLUDE_DIR) + set(USE_DATASKETCHES 1) +endif() + +endif() + +message (STATUS "Using datasketches=${USE_DATASKETCHES}: ${DATASKETCHES_INCLUDE_DIR} : ${DATASKETCHES_LIBRARY}") \ No newline at end of file diff --git a/contrib/datasketches-cpp b/contrib/datasketches-cpp new file mode 160000 index 00000000000..c1a6f8edb49 --- /dev/null +++ b/contrib/datasketches-cpp @@ -0,0 +1 @@ +Subproject commit c1a6f8edb49699520f248d3d02019b87429b4241 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 215a13cce1a..5da0e3af890 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -456,6 +456,10 @@ if (USE_LIBPQXX) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${LIBPQXX_INCLUDE_DIR}) endif() +if (USE_DATASKETCHES) + target_include_directories (clickhouse_aggregate_functions SYSTEM BEFORE PRIVATE ${DATASKETCHES_INCLUDE_DIR}) +endif () + dbms_target_link_libraries(PRIVATE _boost_context) if (ENABLE_TESTS AND USE_GTEST) From 7434022b89dd264485b23b71d2b6ecf39e826017 Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Wed, 3 Mar 2021 22:33:01 +0300 Subject: [PATCH 014/464] still draft needs check --- docs/en/sql-reference/functions/string-search-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 471768da414..4415ee42e2e 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -14,8 +14,8 @@ The search is case-sensitive by default in all these functions. There are separa Returns the position (in bytes) of the found substring in the string, starting from 1. -Function position now supports position(needle in haystack) synax for SQL compatibility. -Enhanced function position to support SQL compatibility syntax: position(needle in haystack), which is same as position(haystack, needle). +!!! note + Now it supports position(needle in haystack) syntax for SQL-compatibility, which is same as position(haystack, needle). This new function POSITION(needle IN haystack) works exactly same as POSITION(haystack, needle). For a case-insensitive search, use the function [positionCaseInsensitive](#positioncaseinsensitive). From 93b661ad5a39bd94e1d85c4b971ee32111bfde7b Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 3 Mar 2021 16:36:20 +0800 Subject: [PATCH 015/464] partition id pruning --- src/Functions/partitionID.cpp | 75 +++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 + src/Functions/ya.make | 1 + src/Interpreters/ExpressionAnalyzer.cpp | 6 +- src/Interpreters/ExpressionAnalyzer.h | 26 +++--- src/Interpreters/TreeRewriter.cpp | 5 +- src/Storages/MergeTree/MergeTreeData.cpp | 33 ++++++++ src/Storages/MergeTree/MergeTreeData.h | 3 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 59 +++++++------ .../MergeTree/MergeTreeDataSelectExecutor.h | 6 ++ src/Storages/StorageMergeTree.cpp | 14 +--- src/Storages/StorageReplicatedMergeTree.cpp | 14 +--- src/Storages/VirtualColumnUtils.cpp | 82 +++++++++++++++---- src/Storages/VirtualColumnUtils.h | 7 +- .../01748_partition_id_pruning.reference | 7 ++ .../01748_partition_id_pruning.sql | 19 +++++ 16 files changed, 278 insertions(+), 81 deletions(-) create mode 100644 src/Functions/partitionID.cpp create mode 100644 tests/queries/0_stateless/01748_partition_id_pruning.reference create mode 100644 tests/queries/0_stateless/01748_partition_id_pruning.sql diff --git a/src/Functions/partitionID.cpp b/src/Functions/partitionID.cpp new file mode 100644 index 00000000000..9f5a3d09b86 --- /dev/null +++ b/src/Functions/partitionID.cpp @@ -0,0 +1,75 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +/** partitionID(x, y, ...) is a function that computes partition ids of arguments. + */ +class FunctionPartitionID : public IFunction +{ +public: + static constexpr auto name = "partitionID"; + + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const override { return true; } + + bool useDefaultImplementationForNulls() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.empty()) + throw Exception("Function " + getName() + " requires at least one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); + } + + virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + size_t size = arguments.size(); + Columns columns; + Block sample_block; + for (const auto & argument : arguments) + { + sample_block.insert(argument); + columns.push_back(argument.column); + } + + auto result_column = ColumnString::create(); + for (size_t j = 0; j < input_rows_count; ++j) + { + Row row(size); + for (size_t i = 0; i < size; ++i) + columns[i]->get(j, row[i]); + MergeTreePartition partition(std::move(row)); + result_column->insert(partition.getID(sample_block)); + } + return result_column; + } +}; + +void registerFunctionPartitionID(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 592f0d6774d..3557b0fb865 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -70,6 +70,7 @@ void registerFunctionTcpPort(FunctionFactory &); void registerFunctionByteSize(FunctionFactory &); void registerFunctionFile(FunctionFactory & factory); void registerFunctionConnectionID(FunctionFactory & factory); +void registerFunctionPartitionID(FunctionFactory & factory); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -140,6 +141,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionByteSize(factory); registerFunctionFile(factory); registerFunctionConnectionID(factory); + registerFunctionPartitionID(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 7a4deae4d04..01857423dc1 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -373,6 +373,7 @@ SRCS( now.cpp now64.cpp nullIf.cpp + partitionID.cpp pi.cpp plus.cpp pointInEllipses.cpp diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 2e2c3354d4c..92388a46872 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -305,7 +305,7 @@ void ExpressionAnalyzer::initGlobalSubqueriesAndExternalTables(bool do_global) } -void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name) +void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name, const SelectQueryOptions & query_options) { auto set_key = PreparedSetKey::forSubquery(*subquery_or_table_name); @@ -341,7 +341,7 @@ void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr prepared_sets[set_key] = std::move(set); } -SetPtr SelectQueryExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name) +SetPtr ExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name) { const auto * table = subquery_or_table_name->as(); if (!table) @@ -387,7 +387,7 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) if (arg->as() || arg->as()) { if (settings.use_index_for_in_with_subqueries) - tryMakeSetForIndexFromSubquery(arg); + tryMakeSetForIndexFromSubquery(arg, query_options); } else { diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index dc7afb183fc..441274200bd 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -127,6 +127,19 @@ public: void makeWindowDescriptions(ActionsDAGPtr actions); + /** + * Create Set from a subquery or a table expression in the query. The created set is suitable for using the index. + * The set will not be created if its size hits the limit. + */ + void tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name, const SelectQueryOptions & query_options = {}); + + /** + * Checks if subquery is not a plain StorageSet. + * Because while making set we will read data from StorageSet which is not allowed. + * Returns valid SetPtr from StorageSet if the latter is used after IN or nullptr otherwise. + */ + SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name); + protected: ExpressionAnalyzer( const ASTPtr & query_, @@ -297,19 +310,6 @@ private: NameSet required_result_columns; SelectQueryOptions query_options; - /** - * Create Set from a subquery or a table expression in the query. The created set is suitable for using the index. - * The set will not be created if its size hits the limit. - */ - void tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name); - - /** - * Checks if subquery is not a plain StorageSet. - * Because while making set we will read data from StorageSet which is not allowed. - * Returns valid SetPtr from StorageSet if the latter is used after IN or nullptr otherwise. - */ - SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name); - /// Create Set-s that we make from IN section to use index on them. void makeSetsForIndex(const ASTPtr & node); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index bcfdf6869c3..5c9ba8fae57 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -656,7 +656,10 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select const auto & partition_desc = metadata_snapshot->getPartitionKey(); if (partition_desc.expression) { - const auto & partition_source_columns = partition_desc.expression->getRequiredColumns(); + auto partition_source_columns = partition_desc.expression->getRequiredColumns(); + partition_source_columns.push_back("_part"); + partition_source_columns.push_back("_partition_id"); + partition_source_columns.push_back("_part_uuid"); optimize_trivial_count = true; for (const auto & required_column : required) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0c22d5fbc0f..49e129e5fac 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -675,6 +676,38 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat } +std::optional MergeTreeData::totalRowsByPartitionPredicateImpl( + const SelectQueryInfo & query_info, const Context & context, const DataPartsVector & parts) const +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + Block part_block = MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns(); + ASTPtr expression_ast; + bool valid = VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, part_block, expression_ast); + PartitionPruner partition_pruner(metadata_snapshot->getPartitionKey(), query_info, context, true /* strict */); + if (partition_pruner.isUseless() && !valid) + return {}; + + std::unordered_set part_values; + if (valid) + { + MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(parts, part_block); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, part_block, context, expression_ast); + part_values = VirtualColumnUtils::extractSingleValueFromBlock(part_block, "_part"); + if (part_values.empty()) + return 0; + } + // At this point, empty `part_values` means all parts. + + size_t res = 0; + for (const auto & part : parts) + { + if ((part_values.empty() || part_values.find(part->name) != part_values.end()) && !partition_pruner.canBePruned(part)) + res += part->rows_count; + } + return res; +} + + String MergeTreeData::MergingParams::getModeName() const { switch (mode) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 15059ab47e5..77df4ac5026 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -825,6 +825,9 @@ protected: return {begin, end}; } + std::optional totalRowsByPartitionPredicateImpl( + const SelectQueryInfo & query_info, const Context & context, const DataPartsVector & parts) const; + static decltype(auto) getStateModifier(DataPartState state) { return [state] (const DataPartPtr & part) { part->setState(state); }; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b1f3f524beb..917c0bcec81 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -71,28 +71,30 @@ MergeTreeDataSelectExecutor::MergeTreeDataSelectExecutor(const MergeTreeData & d } -/// Construct a block consisting only of possible values of virtual columns -static Block getBlockWithVirtualPartColumns(const MergeTreeData::DataPartsVector & parts, bool with_uuid) +Block MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns() { - auto part_column = ColumnString::create(); - auto part_uuid_column = ColumnUUID::create(); + return Block(std::initializer_list{ + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_part"), + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_partition_id"), + ColumnWithTypeAndName(ColumnUUID::create(), std::make_shared(), "_part_uuid")}); +} + +void MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(const MergeTreeData::DataPartsVector & parts, Block & block) +{ + MutableColumns columns = block.mutateColumns(); + + auto & part_column = columns[0]; + auto & partition_id_column = columns[1]; + auto & part_uuid_column = columns[2]; for (const auto & part : parts) { part_column->insert(part->name); - if (with_uuid) - part_uuid_column->insert(part->uuid); + partition_id_column->insert(part->info.partition_id); + part_uuid_column->insert(part->uuid); } - if (with_uuid) - { - return Block(std::initializer_list{ - ColumnWithTypeAndName(std::move(part_column), std::make_shared(), "_part"), - ColumnWithTypeAndName(std::move(part_uuid_column), std::make_shared(), "_part_uuid"), - }); - } - - return Block{ColumnWithTypeAndName(std::move(part_column), std::make_shared(), "_part")}; + block.setColumns(std::move(columns)); } @@ -176,8 +178,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( Names real_column_names; size_t total_parts = parts.size(); - bool part_column_queried = false; - bool part_uuid_column_queried = false; bool sample_factor_column_queried = false; Float64 used_sample_factor = 1; @@ -186,7 +186,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( { if (name == "_part") { - part_column_queried = true; virt_column_names.push_back(name); } else if (name == "_part_index") @@ -199,7 +198,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( } else if (name == "_part_uuid") { - part_uuid_column_queried = true; virt_column_names.push_back(name); } else if (name == "_sample_factor") @@ -219,12 +217,21 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( if (real_column_names.empty()) real_column_names.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); - /// If `_part` or `_part_uuid` virtual columns are requested, we try to filter out data by them. - Block virtual_columns_block = getBlockWithVirtualPartColumns(parts, part_uuid_column_queried); - if (part_column_queried || part_uuid_column_queried) - VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context); + std::unordered_set part_values; + ASTPtr expression_ast; + Block virtual_columns_block = getSampleBlockWithVirtualPartColumns(); + VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, virtual_columns_block, expression_ast); - auto part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); + // If there is still something left, fill the virtual block and do the filtering. + if (expression_ast) + { + fillBlockWithVirtualPartColumns(parts, virtual_columns_block); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); + part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); + if (part_values.empty()) + return {}; + } + // At this point, empty `part_values` means all parts. metadata_snapshot->check(real_column_names, data.getVirtuals(), data.getStorageID()); @@ -1899,7 +1906,7 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( for (const auto & part : prev_parts) { - if (part_values.find(part->name) == part_values.end()) + if (!part_values.empty() && part_values.find(part->name) == part_values.end()) continue; if (part->isEmpty()) @@ -1950,7 +1957,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( for (const auto & part : prev_parts) { - if (part_values.find(part->name) == part_values.end()) + if (!part_values.empty() && part_values.find(part->name) == part_values.end()) continue; if (part->isEmpty()) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 634719639ad..0702605a539 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -44,6 +44,12 @@ public: unsigned num_streams, const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr) const; + /// Construct a sample block consisting only of possible virtual columns for part pruning. + static Block getSampleBlockWithVirtualPartColumns(); + + /// Fill in values of possible virtual columns for part pruning. + static void fillBlockWithVirtualPartColumns(const MergeTreeData::DataPartsVector & parts, Block & block); + private: const MergeTreeData & data; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c8f44c78e6e..ce81849708b 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -208,18 +208,8 @@ std::optional StorageMergeTree::totalRows(const Settings &) const std::optional StorageMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const { - auto metadata_snapshot = getInMemoryMetadataPtr(); - PartitionPruner partition_pruner(metadata_snapshot->getPartitionKey(), query_info, context, true /* strict */); - if (partition_pruner.isUseless()) - return {}; - size_t res = 0; - auto lock = lockParts(); - for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) - { - if (!partition_pruner.canBePruned(part)) - res += part->rows_count; - } - return res; + auto parts = getDataPartsVector({DataPartState::Committed}); + return totalRowsByPartitionPredicateImpl(query_info, context, parts); } std::optional StorageMergeTree::totalBytes(const Settings &) const diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 68f3b6d80d1..9f18d9ead21 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3890,17 +3890,9 @@ std::optional StorageReplicatedMergeTree::totalRows(const Settings & set std::optional StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const { - auto metadata_snapshot = getInMemoryMetadataPtr(); - PartitionPruner partition_pruner(metadata_snapshot->getPartitionKey(), query_info, context, true /* strict */); - if (partition_pruner.isUseless()) - return {}; - size_t res = 0; - foreachCommittedParts([&](auto & part) - { - if (!partition_pruner.canBePruned(part)) - res += part->rows_count; - }, context.getSettingsRef().select_sequential_consistency); - return res; + DataPartsVector parts; + foreachCommittedParts([&](auto & part) { parts.push_back(part); }, context.getSettingsRef().select_sequential_consistency); + return totalRowsByPartitionPredicateImpl(query_info, context, parts); } std::optional StorageReplicatedMergeTree::totalBytes(const Settings & settings) const diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 6b99dc25e37..ba35bde6d51 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -5,12 +5,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -30,29 +32,52 @@ namespace /// Verifying that the function depends only on the specified columns bool isValidFunction(const ASTPtr & expression, const NameSet & columns) { - for (const auto & child : expression->children) - if (!isValidFunction(child, columns)) - return false; - - if (auto opt_name = IdentifierSemantic::getColumnName(expression)) - return columns.count(*opt_name); + const auto * function = expression->as(); + if (function) + { + if (functionIsInOrGlobalInOperator(function->name)) + { + // Second argument of IN can be a scalar subquery + if (!isValidFunction(function->arguments->children[0], columns)) + return false; + } + else + { + if (function->arguments) + { + for (const auto & child : function->arguments->children) + if (!isValidFunction(child, columns)) + return false; + } + } + } + else + { + if (auto opt_name = IdentifierSemantic::getColumnName(expression)) + return columns.count(*opt_name); + } return true; } /// Extract all subfunctions of the main conjunction, but depending only on the specified columns -void extractFunctions(const ASTPtr & expression, const NameSet & columns, std::vector & result) +bool extractFunctions(const ASTPtr & expression, const NameSet & columns, std::vector & result) { const auto * function = expression->as(); - if (function && function->name == "and") + if (function && (function->name == "and" || function->name == "indexHint")) { + bool ret = true; for (const auto & child : function->arguments->children) - extractFunctions(child, columns, result); + ret &= extractFunctions(child, columns, result); + return ret; } else if (isValidFunction(expression, columns)) { result.push_back(expression->clone()); + return true; } + else + return false; } /// Construct a conjunction from given functions @@ -65,6 +90,25 @@ ASTPtr buildWhereExpression(const ASTs & functions) return makeASTFunction("and", functions); } +void buildSets(const ASTPtr & expression, ExpressionAnalyzer & analyzer) +{ + const auto * func = expression->as(); + if (func && functionIsInOrGlobalInOperator(func->name)) + { + const IAST & args = *func->arguments; + const ASTPtr & arg = args.children.at(1); + if (arg->as() || arg->as()) + { + analyzer.tryMakeSetForIndexFromSubquery(arg); + } + } + else + { + for (const auto & child : expression->children) + buildSets(child, analyzer); + } +} + } namespace VirtualColumnUtils @@ -96,11 +140,12 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va } } -void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context) +bool prepareFilterBlockWithQuery(const ASTPtr & query, const Block & block, ASTPtr & expression_ast) { + bool ret = true; const auto & select = query->as(); if (!select.where() && !select.prewhere()) - return; + return ret; NameSet columns; for (const auto & it : block.getNamesAndTypesList()) @@ -109,17 +154,26 @@ void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & c /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. std::vector functions; if (select.where()) - extractFunctions(select.where(), columns, functions); + ret &= extractFunctions(select.where(), columns, functions); if (select.prewhere()) - extractFunctions(select.prewhere(), columns, functions); + ret &= extractFunctions(select.prewhere(), columns, functions); + + expression_ast = buildWhereExpression(functions); + return ret; +} + +void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast) +{ + if (!expression_ast) + prepareFilterBlockWithQuery(query, block, expression_ast); - ASTPtr expression_ast = buildWhereExpression(functions); if (!expression_ast) return; /// Let's analyze and calculate the expression. auto syntax_result = TreeRewriter(context).analyze(expression_ast, block.getNamesAndTypesList()); ExpressionAnalyzer analyzer(expression_ast, syntax_result, context); + buildSets(expression_ast, analyzer); ExpressionActionsPtr actions = analyzer.getActions(false); Block block_with_filter = block; diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 445a996ab87..326738f4259 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -24,9 +24,14 @@ namespace VirtualColumnUtils /// - `WITH toUInt16(9000) as _port`. void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & value, const String & func = ""); +/// Prepare `expression_ast` to filter block. Returns true if `expression_ast` is not trimmed, that is, +/// the sample block provides all needed columns, else return false. +bool prepareFilterBlockWithQuery(const ASTPtr & query, const Block & sample_block, ASTPtr & expression_ast); + /// Leave in the block only the rows that fit under the WHERE clause and the PREWHERE clause of the query. /// Only elements of the outer conjunction are considered, depending only on the columns present in the block. -void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context); +/// If `expression_ast` is passed, use it to filter block. +void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast = {}); /// Extract from the input stream a set of `name` column values template diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.reference b/tests/queries/0_stateless/01748_partition_id_pruning.reference new file mode 100644 index 00000000000..192e33c03c9 --- /dev/null +++ b/tests/queries/0_stateless/01748_partition_id_pruning.reference @@ -0,0 +1,7 @@ +1 1 +1 2 +1 3 +1 1 +1 2 +1 3 +3 diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql new file mode 100644 index 00000000000..cca366c5168 --- /dev/null +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -0,0 +1,19 @@ +drop table if exists x; + +create table x (i int, j int) engine MergeTree partition by i order by j settings index_granularity = 1; + +insert into x values (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); + +set max_rows_to_read = 3; + +select * from x where _partition_id = partitionID(1); + +set max_rows_to_read = 4; -- one row for subquery + +select * from x where _partition_id in (select partitionID(number + 1) from numbers(1)); + +-- trivial count optimization test +set max_rows_to_read = 1; -- one row for subquery +select count() from x where _partition_id in (select partitionID(number + 1) from numbers(1)); + +drop table x; From 2f8f4e96973a48b1900e08cee6b0094381713542 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 3 Mar 2021 22:26:55 +0800 Subject: [PATCH 016/464] Fix tests --- src/Storages/VirtualColumnUtils.cpp | 4 ++++ tests/queries/0_stateless/01651_bugs_from_15889.reference | 1 + tests/queries/0_stateless/01651_bugs_from_15889.sql | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index ba35bde6d51..196455c70f0 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -41,6 +41,10 @@ bool isValidFunction(const ASTPtr & expression, const NameSet & columns) if (!isValidFunction(function->arguments->children[0], columns)) return false; } + else if (function->name == "ignore") + { + return false; + } else { if (function->arguments) diff --git a/tests/queries/0_stateless/01651_bugs_from_15889.reference b/tests/queries/0_stateless/01651_bugs_from_15889.reference index 77ac542d4fb..28271a697e2 100644 --- a/tests/queries/0_stateless/01651_bugs_from_15889.reference +++ b/tests/queries/0_stateless/01651_bugs_from_15889.reference @@ -1,2 +1,3 @@ 0 +0 diff --git a/tests/queries/0_stateless/01651_bugs_from_15889.sql b/tests/queries/0_stateless/01651_bugs_from_15889.sql index 1fbf669a1b8..97da4d78ab6 100644 --- a/tests/queries/0_stateless/01651_bugs_from_15889.sql +++ b/tests/queries/0_stateless/01651_bugs_from_15889.sql @@ -8,7 +8,7 @@ INSERT INTO xp SELECT '2020-01-01', number, '' FROM numbers(100000); CREATE TABLE xp_d AS xp ENGINE = Distributed(test_shard_localhost, currentDatabase(), xp); -SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- { serverError 20 } +SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- B > NULL is evaluated to 0 and this works SELECT count() FROM xp_d WHERE A GLOBAL IN (SELECT NULL); -- { serverError 53 } From 4c9b0f666ca165d51dd7c9f395b5fc1da8fb9485 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 3 Mar 2021 23:32:37 +0800 Subject: [PATCH 017/464] Fix error --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 917c0bcec81..0d4c4c4aa9a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -229,7 +229,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); if (part_values.empty()) - return {}; + return std::make_unique(); } // At this point, empty `part_values` means all parts. @@ -380,7 +380,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( { LOG_DEBUG(log, "Will use no data on this replica because parallel replicas processing has been requested" " (the setting 'max_parallel_replicas') but the table does not support sampling and this replica is not the first."); - return {}; + return std::make_unique(); } bool use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); From 9205fad8c762644d374936dbba39a3730118dcea Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 4 Mar 2021 13:59:57 +0800 Subject: [PATCH 018/464] Better --- .../ExecuteScalarSubqueriesVisitor.cpp | 2 +- src/Interpreters/ExpressionAnalyzer.cpp | 4 +- src/Interpreters/ExpressionAnalyzer.h | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 15 +++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 18 +++--- src/Storages/StorageMerge.cpp | 9 +-- src/Storages/StorageMerge.h | 2 +- src/Storages/System/StorageSystemColumns.cpp | 4 +- .../System/StorageSystemDistributionQueue.cpp | 2 +- .../System/StorageSystemMutations.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 4 +- src/Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- src/Storages/System/StorageSystemTables.cpp | 6 +- src/Storages/VirtualColumnUtils.cpp | 59 +++++++------------ src/Storages/VirtualColumnUtils.h | 7 ++- ...read_distribution_and_max_rows_to_read.sql | 5 +- 17 files changed, 68 insertions(+), 77 deletions(-) diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 7ee7bb1f301..94b3a5218b4 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -201,7 +201,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr void ExecuteScalarSubqueriesMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data & data) { /// Don't descend into subqueries in arguments of IN operator. - /// But if an argument is not subquery, than deeper may be scalar subqueries and we need to descend in them. + /// But if an argument is not subquery, then deeper may be scalar subqueries and we need to descend in them. std::vector out; if (checkFunctionIsInOrGlobalInOperator(func)) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 92388a46872..50dd6ede2a1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1324,9 +1324,9 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool proje } -ExpressionActionsPtr ExpressionAnalyzer::getConstActions() +ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const NamesAndTypesList & constant_inputs) { - auto actions = std::make_shared(NamesAndTypesList()); + auto actions = std::make_shared(constant_inputs); getRootActions(query, true, actions, true); return std::make_shared(actions); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 441274200bd..c44507cc4e2 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -110,7 +110,7 @@ public: /// Actions that can be performed on an empty block: adding constants and applying functions that depend only on constants. /// Does not execute subqueries. - ExpressionActionsPtr getConstActions(); + ExpressionActionsPtr getConstActions(const NamesAndTypesList & constant_inputs = {}); /** Sets that require a subquery to be create. * Only the sets needed to perform actions returned from already executed `append*` or `getActions`. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 49e129e5fac..945fb5639c1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -680,19 +680,22 @@ std::optional MergeTreeData::totalRowsByPartitionPredicateImpl( const SelectQueryInfo & query_info, const Context & context, const DataPartsVector & parts) const { auto metadata_snapshot = getInMemoryMetadataPtr(); - Block part_block = MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns(); ASTPtr expression_ast; - bool valid = VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, part_block, expression_ast); + Block virtual_columns_block = MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns(); + + // Generate valid expressions for filtering + bool valid = VirtualColumnUtils::prepareFilterBlockWithQuery(query_info, context, virtual_columns_block, expression_ast); + PartitionPruner partition_pruner(metadata_snapshot->getPartitionKey(), query_info, context, true /* strict */); if (partition_pruner.isUseless() && !valid) return {}; std::unordered_set part_values; - if (valid) + if (valid && expression_ast) { - MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(parts, part_block); - VirtualColumnUtils::filterBlockWithQuery(query_info.query, part_block, context, expression_ast); - part_values = VirtualColumnUtils::extractSingleValueFromBlock(part_block, "_part"); + MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(parts, virtual_columns_block); + VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, context, expression_ast); + part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); if (part_values.empty()) return 0; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 0d4c4c4aa9a..59bf32fc14a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -39,6 +39,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -74,14 +75,15 @@ MergeTreeDataSelectExecutor::MergeTreeDataSelectExecutor(const MergeTreeData & d Block MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns() { return Block(std::initializer_list{ - ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_part"), - ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_partition_id"), - ColumnWithTypeAndName(ColumnUUID::create(), std::make_shared(), "_part_uuid")}); + ColumnWithTypeAndName(DataTypeString().createColumnConstWithDefaultValue(1), std::make_shared(), "_part"), + ColumnWithTypeAndName(DataTypeString().createColumnConstWithDefaultValue(1), std::make_shared(), "_partition_id"), + ColumnWithTypeAndName(DataTypeUUID().createColumnConstWithDefaultValue(1), std::make_shared(), "_part_uuid")}); } void MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(const MergeTreeData::DataPartsVector & parts, Block & block) { - MutableColumns columns = block.mutateColumns(); + materializeBlockInplace(block); + MutableColumns columns = block.cloneEmptyColumns(); auto & part_column = columns[0]; auto & partition_id_column = columns[1]; @@ -219,14 +221,16 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( std::unordered_set part_values; ASTPtr expression_ast; - Block virtual_columns_block = getSampleBlockWithVirtualPartColumns(); - VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, virtual_columns_block, expression_ast); + auto virtual_columns_block = getSampleBlockWithVirtualPartColumns(); + + // Generate valid expressions for filtering + VirtualColumnUtils::prepareFilterBlockWithQuery(query_info, context, virtual_columns_block, expression_ast); // If there is still something left, fill the virtual block and do the filtering. if (expression_ast) { fillBlockWithVirtualPartColumns(parts, virtual_columns_block); - VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); + VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, context, expression_ast); part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); if (part_values.empty()) return std::make_unique(); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 46be91ba258..54f87d4e459 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -219,8 +219,8 @@ Pipe StorageMerge::read( /** First we make list of selected tables to find out its size. * This is necessary to correctly pass the recommended number of threads to each table. */ - StorageListWithLocks selected_tables = getSelectedTables( - query_info.query, has_table_virtual_column, context.getCurrentQueryId(), context.getSettingsRef()); + StorageListWithLocks selected_tables + = getSelectedTables(query_info, has_table_virtual_column, context.getCurrentQueryId(), context.getSettingsRef()); if (selected_tables.empty()) /// FIXME: do we support sampling in this case? @@ -407,8 +407,9 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( - const ASTPtr & query, bool has_virtual_column, const String & query_id, const Settings & settings) const + const SelectQueryInfo & query_info, bool has_virtual_column, const String & query_id, const Settings & settings) const { + const ASTPtr & query = query_info.query; StorageListWithLocks selected_tables; DatabaseTablesIteratorPtr iterator = getDatabaseIterator(global_context); @@ -436,7 +437,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( if (has_virtual_column) { Block virtual_columns_block = Block{ColumnWithTypeAndName(std::move(virtual_column), std::make_shared(), "_table")}; - VirtualColumnUtils::filterBlockWithQuery(query, virtual_columns_block, global_context); + VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, global_context); auto values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_table"); /// Remove unused tables from the list diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index eaffd34a379..ea8667aa186 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -59,7 +59,7 @@ private: StorageListWithLocks getSelectedTables(const String & query_id, const Settings & settings) const; StorageMerge::StorageListWithLocks getSelectedTables( - const ASTPtr & query, bool has_virtual_column, const String & query_id, const Settings & settings) const; + const SelectQueryInfo & query_info, bool has_virtual_column, const String & query_id, const Settings & settings) const; template StoragePtr getFirstTable(F && predicate) const; diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 6726d502071..75d90ffe1f8 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -299,7 +299,7 @@ Pipe StorageSystemColumns::read( block_to_filter.insert(ColumnWithTypeAndName(std::move(database_column_mut), std::make_shared(), "database")); /// Filter block with `database` column. - VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); if (!block_to_filter.rows()) { @@ -345,7 +345,7 @@ Pipe StorageSystemColumns::read( } /// Filter block with `database` and `table` columns. - VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); if (!block_to_filter.rows()) { diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index db649e7e1ba..58b468ab7f6 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -155,7 +155,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons { col_table_to_filter, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index f66f57ef5d1..50498f49c81 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -89,7 +89,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex { col_table, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 39cc651e147..e88cf8b2865 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -93,7 +93,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const std::move(database_column_mut), std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. - VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); rows = block_to_filter.rows(); /// Block contains new columns, update database_column. @@ -162,7 +162,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const if (rows) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); rows = block_to_filter.rows(); } diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 0af67ab6986..b66e578ea1a 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -135,7 +135,7 @@ Pipe StorageSystemReplicas::read( { col_engine, std::make_shared(), "engine" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); if (!filtered_block.rows()) return {}; diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 9cd5e8b8ff3..549b93b5ebc 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -98,7 +98,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const { col_table_to_filter, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 132ed234323..af25d299c2e 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -62,7 +62,7 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) } -static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & context) +static ColumnPtr getFilteredDatabases(const SelectQueryInfo & query_info, const Context & context) { MutableColumnPtr column = ColumnString::create(); @@ -76,7 +76,7 @@ static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & cont } Block block { ColumnWithTypeAndName(std::move(column), std::make_shared(), "database") }; - VirtualColumnUtils::filterBlockWithQuery(query, block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info, block, context); return block.getByPosition(0).column; } @@ -524,7 +524,7 @@ Pipe StorageSystemTables::read( } } - ColumnPtr filtered_databases_column = getFilteredDatabases(query_info.query, context); + ColumnPtr filtered_databases_column = getFilteredDatabases(query_info, context); return Pipe(std::make_shared( std::move(columns_mask), std::move(res_block), max_block_size, std::move(filtered_databases_column), context)); diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 196455c70f0..af817c52805 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -30,52 +30,33 @@ namespace { /// Verifying that the function depends only on the specified columns -bool isValidFunction(const ASTPtr & expression, const NameSet & columns) +bool isValidFunction(const ASTPtr & expression, const Block & block) { const auto * function = expression->as(); - if (function) + if (function && functionIsInOrGlobalInOperator(function->name)) { - if (functionIsInOrGlobalInOperator(function->name)) - { - // Second argument of IN can be a scalar subquery - if (!isValidFunction(function->arguments->children[0], columns)) - return false; - } - else if (function->name == "ignore") - { - return false; - } - else - { - if (function->arguments) - { - for (const auto & child : function->arguments->children) - if (!isValidFunction(child, columns)) - return false; - } - } + // Second argument of IN can be a scalar subquery + return isValidFunction(function->arguments->children[0], block); } else { - if (auto opt_name = IdentifierSemantic::getColumnName(expression)) - return columns.count(*opt_name); + auto column_name = expression->getColumnName(); + return block.has(column_name) && isColumnConst(*block.getByName(column_name).column); } - - return true; } /// Extract all subfunctions of the main conjunction, but depending only on the specified columns -bool extractFunctions(const ASTPtr & expression, const NameSet & columns, std::vector & result) +bool extractFunctions(const ASTPtr & expression, const Block & block, std::vector & result) { const auto * function = expression->as(); if (function && (function->name == "and" || function->name == "indexHint")) { bool ret = true; for (const auto & child : function->arguments->children) - ret &= extractFunctions(child, columns, result); + ret &= extractFunctions(child, block, result); return ret; } - else if (isValidFunction(expression, columns)) + else if (isValidFunction(expression, block)) { result.push_back(expression->clone()); return true; @@ -124,7 +105,6 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va if (!select.with()) select.setExpression(ASTSelectQuery::Expression::WITH, std::make_shared()); - if (func.empty()) { auto literal = std::make_shared(value); @@ -144,37 +124,38 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va } } -bool prepareFilterBlockWithQuery(const ASTPtr & query, const Block & block, ASTPtr & expression_ast) +bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Context & context, Block block, ASTPtr & expression_ast) { bool ret = true; - const auto & select = query->as(); + const auto & select = query_info.query->as(); if (!select.where() && !select.prewhere()) return ret; - NameSet columns; - for (const auto & it : block.getNamesAndTypesList()) - columns.insert(it.name); + // Prepare a block with valid expressions + ExpressionAnalyzer(query_info.query, query_info.syntax_analyzer_result, context) + .getConstActions(block.getNamesAndTypesList()) + ->execute(block); /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. std::vector functions; if (select.where()) - ret &= extractFunctions(select.where(), columns, functions); + ret &= extractFunctions(select.where(), block, functions); if (select.prewhere()) - ret &= extractFunctions(select.prewhere(), columns, functions); + ret &= extractFunctions(select.prewhere(), block, functions); expression_ast = buildWhereExpression(functions); return ret; } -void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast) +void filterBlockWithQuery(const SelectQueryInfo & query_info, Block & block, const Context & context, ASTPtr expression_ast) { if (!expression_ast) - prepareFilterBlockWithQuery(query, block, expression_ast); + prepareFilterBlockWithQuery(query_info, context, block, expression_ast); if (!expression_ast) return; - /// Let's analyze and calculate the expression. + /// Let's analyze and calculate the prepared expression. auto syntax_result = TreeRewriter(context).analyze(expression_ast, block.getNamesAndTypesList()); ExpressionAnalyzer analyzer(expression_ast, syntax_result, context); buildSets(expression_ast, analyzer); diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 326738f4259..7c414aa0984 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB @@ -25,13 +26,13 @@ namespace VirtualColumnUtils void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & value, const String & func = ""); /// Prepare `expression_ast` to filter block. Returns true if `expression_ast` is not trimmed, that is, -/// the sample block provides all needed columns, else return false. -bool prepareFilterBlockWithQuery(const ASTPtr & query, const Block & sample_block, ASTPtr & expression_ast); +/// `block` provides all needed columns for `expression_ast`, else return false. +bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Context & context, Block block, ASTPtr & expression_ast); /// Leave in the block only the rows that fit under the WHERE clause and the PREWHERE clause of the query. /// Only elements of the outer conjunction are considered, depending only on the columns present in the block. /// If `expression_ast` is passed, use it to filter block. -void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast = {}); +void filterBlockWithQuery(const SelectQueryInfo & query_info, Block & block, const Context & context, ASTPtr expression_ast = {}); /// Extract from the input stream a set of `name` column values template diff --git a/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql b/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql index d31989d65bd..5abb1af620a 100644 --- a/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql +++ b/tests/queries/0_stateless/00971_merge_tree_uniform_read_distribution_and_max_rows_to_read.sql @@ -10,7 +10,8 @@ SELECT count() FROM merge_tree; SET max_rows_to_read = 900000; -SELECT count() FROM merge_tree WHERE not ignore(); -- { serverError 158 } -SELECT count() FROM merge_tree WHERE not ignore(); -- { serverError 158 } +-- constant ignore will be pruned by part pruner. ignore(*) is used. +SELECT count() FROM merge_tree WHERE not ignore(*); -- { serverError 158 } +SELECT count() FROM merge_tree WHERE not ignore(*); -- { serverError 158 } DROP TABLE merge_tree; From fac832227cd08c7e12170c4ca60fbc33a2b5b7e0 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 4 Mar 2021 16:53:31 +0800 Subject: [PATCH 019/464] Fix again --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/ExpressionAnalyzer.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 9 ++++----- src/Storages/VirtualColumnUtils.cpp | 13 +++++++++++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 50dd6ede2a1..f931ecf8c21 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1324,7 +1324,7 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool proje } -ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const NamesAndTypesList & constant_inputs) +ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const ColumnsWithTypeAndName & constant_inputs) { auto actions = std::make_shared(constant_inputs); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index c44507cc4e2..69b6d75de35 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -110,7 +110,7 @@ public: /// Actions that can be performed on an empty block: adding constants and applying functions that depend only on constants. /// Does not execute subqueries. - ExpressionActionsPtr getConstActions(const NamesAndTypesList & constant_inputs = {}); + ExpressionActionsPtr getConstActions(const ColumnsWithTypeAndName & constant_inputs = {}); /** Sets that require a subquery to be create. * Only the sets needed to perform actions returned from already executed `append*` or `getActions`. diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 59bf32fc14a..2b26c59ee08 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -75,15 +75,14 @@ MergeTreeDataSelectExecutor::MergeTreeDataSelectExecutor(const MergeTreeData & d Block MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns() { return Block(std::initializer_list{ - ColumnWithTypeAndName(DataTypeString().createColumnConstWithDefaultValue(1), std::make_shared(), "_part"), - ColumnWithTypeAndName(DataTypeString().createColumnConstWithDefaultValue(1), std::make_shared(), "_partition_id"), - ColumnWithTypeAndName(DataTypeUUID().createColumnConstWithDefaultValue(1), std::make_shared(), "_part_uuid")}); + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_part"), + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "_partition_id"), + ColumnWithTypeAndName(ColumnUUID::create(), std::make_shared(), "_part_uuid")}); } void MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(const MergeTreeData::DataPartsVector & parts, Block & block) { - materializeBlockInplace(block); - MutableColumns columns = block.cloneEmptyColumns(); + MutableColumns columns = block.mutateColumns(); auto & part_column = columns[0]; auto & partition_id_column = columns[1]; diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index af817c52805..c69fa59d93a 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -131,9 +131,18 @@ bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Conte if (!select.where() && !select.prewhere()) return ret; + ASTPtr condition_ast; + if (select.prewhere() && select.where()) + condition_ast = makeASTFunction("and", select.prewhere()->clone(), select.where()->clone()); + else + condition_ast = select.prewhere() ? select.prewhere()->clone() : select.where()->clone(); + // Prepare a block with valid expressions - ExpressionAnalyzer(query_info.query, query_info.syntax_analyzer_result, context) - .getConstActions(block.getNamesAndTypesList()) + for (size_t i = 0; i < block.columns(); ++i) + block.getByPosition(i).column = block.getByPosition(i).type->createColumnConstWithDefaultValue(1); + + ExpressionAnalyzer(condition_ast, query_info.syntax_analyzer_result, context) + .getConstActions(block.getColumnsWithTypeAndName()) ->execute(block); /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. From 909cb3c24324e923a371255e01cbba6f0730b854 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 4 Mar 2021 22:27:07 +0800 Subject: [PATCH 020/464] Fix again.... --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- src/Storages/StorageMerge.cpp | 2 +- src/Storages/System/StorageSystemColumns.cpp | 4 ++-- .../System/StorageSystemDistributionQueue.cpp | 2 +- .../System/StorageSystemMutations.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 4 ++-- src/Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- src/Storages/System/StorageSystemTables.cpp | 2 +- src/Storages/VirtualColumnUtils.cpp | 21 ++++++++++++------- src/Storages/VirtualColumnUtils.h | 4 ++-- 12 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 945fb5639c1..e7b83eae85d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -684,7 +684,7 @@ std::optional MergeTreeData::totalRowsByPartitionPredicateImpl( Block virtual_columns_block = MergeTreeDataSelectExecutor::getSampleBlockWithVirtualPartColumns(); // Generate valid expressions for filtering - bool valid = VirtualColumnUtils::prepareFilterBlockWithQuery(query_info, context, virtual_columns_block, expression_ast); + bool valid = VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, context, virtual_columns_block, expression_ast); PartitionPruner partition_pruner(metadata_snapshot->getPartitionKey(), query_info, context, true /* strict */); if (partition_pruner.isUseless() && !valid) @@ -694,7 +694,7 @@ std::optional MergeTreeData::totalRowsByPartitionPredicateImpl( if (valid && expression_ast) { MergeTreeDataSelectExecutor::fillBlockWithVirtualPartColumns(parts, virtual_columns_block); - VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, context, expression_ast); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); if (part_values.empty()) return 0; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 2b26c59ee08..6588729e25e 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -223,13 +223,13 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( auto virtual_columns_block = getSampleBlockWithVirtualPartColumns(); // Generate valid expressions for filtering - VirtualColumnUtils::prepareFilterBlockWithQuery(query_info, context, virtual_columns_block, expression_ast); + VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, context, virtual_columns_block, expression_ast); // If there is still something left, fill the virtual block and do the filtering. if (expression_ast) { fillBlockWithVirtualPartColumns(parts, virtual_columns_block); - VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, context, expression_ast); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); if (part_values.empty()) return std::make_unique(); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 54f87d4e459..6ac8d02e54d 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -437,7 +437,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables( if (has_virtual_column) { Block virtual_columns_block = Block{ColumnWithTypeAndName(std::move(virtual_column), std::make_shared(), "_table")}; - VirtualColumnUtils::filterBlockWithQuery(query_info, virtual_columns_block, global_context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, global_context); auto values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_table"); /// Remove unused tables from the list diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 75d90ffe1f8..6726d502071 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -299,7 +299,7 @@ Pipe StorageSystemColumns::read( block_to_filter.insert(ColumnWithTypeAndName(std::move(database_column_mut), std::make_shared(), "database")); /// Filter block with `database` column. - VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); if (!block_to_filter.rows()) { @@ -345,7 +345,7 @@ Pipe StorageSystemColumns::read( } /// Filter block with `database` and `table` columns. - VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); if (!block_to_filter.rows()) { diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 58b468ab7f6..db649e7e1ba 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -155,7 +155,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons { col_table_to_filter, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 50498f49c81..f66f57ef5d1 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -89,7 +89,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex { col_table, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index e88cf8b2865..39cc651e147 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -93,7 +93,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const std::move(database_column_mut), std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. - VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); rows = block_to_filter.rows(); /// Block contains new columns, update database_column. @@ -162,7 +162,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const if (rows) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithQuery(query_info, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); rows = block_to_filter.rows(); } diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index b66e578ea1a..0af67ab6986 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -135,7 +135,7 @@ Pipe StorageSystemReplicas::read( { col_engine, std::make_shared(), "engine" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return {}; diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 549b93b5ebc..9cd5e8b8ff3 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -98,7 +98,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const { col_table_to_filter, std::make_shared(), "table" }, }; - VirtualColumnUtils::filterBlockWithQuery(query_info, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index af25d299c2e..2b1788f991d 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -76,7 +76,7 @@ static ColumnPtr getFilteredDatabases(const SelectQueryInfo & query_info, const } Block block { ColumnWithTypeAndName(std::move(column), std::make_shared(), "database") }; - VirtualColumnUtils::filterBlockWithQuery(query_info, block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block, context); return block.getByPosition(0).column; } diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index c69fa59d93a..537ef91f4f5 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace DB @@ -124,10 +125,10 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va } } -bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Context & context, Block block, ASTPtr & expression_ast) +bool prepareFilterBlockWithQuery(const ASTPtr & query, const Context & context, Block block, ASTPtr & expression_ast) { bool ret = true; - const auto & select = query_info.query->as(); + const auto & select = query->as(); if (!select.where() && !select.prewhere()) return ret; @@ -141,9 +142,15 @@ bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Conte for (size_t i = 0; i < block.columns(); ++i) block.getByPosition(i).column = block.getByPosition(i).type->createColumnConstWithDefaultValue(1); - ExpressionAnalyzer(condition_ast, query_info.syntax_analyzer_result, context) - .getConstActions(block.getColumnsWithTypeAndName()) - ->execute(block); + auto actions = std::make_shared(block.getColumnsWithTypeAndName()); + PreparedSets prepared_sets; + SubqueriesForSets subqueries_for_sets; + ActionsVisitor::Data visitor_data( + context, SizeLimits{}, 1, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); + ActionsVisitor(visitor_data).visit(condition_ast); + actions = visitor_data.getActions(); + auto expression_actions = std::make_shared(actions); + expression_actions->execute(block); /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. std::vector functions; @@ -156,10 +163,10 @@ bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Conte return ret; } -void filterBlockWithQuery(const SelectQueryInfo & query_info, Block & block, const Context & context, ASTPtr expression_ast) +void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast) { if (!expression_ast) - prepareFilterBlockWithQuery(query_info, context, block, expression_ast); + prepareFilterBlockWithQuery(query, context, block, expression_ast); if (!expression_ast) return; diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 7c414aa0984..78e3d62472e 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -27,12 +27,12 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va /// Prepare `expression_ast` to filter block. Returns true if `expression_ast` is not trimmed, that is, /// `block` provides all needed columns for `expression_ast`, else return false. -bool prepareFilterBlockWithQuery(const SelectQueryInfo & query_info, const Context & context, Block block, ASTPtr & expression_ast); +bool prepareFilterBlockWithQuery(const ASTPtr & query, const Context & context, Block block, ASTPtr & expression_ast); /// Leave in the block only the rows that fit under the WHERE clause and the PREWHERE clause of the query. /// Only elements of the outer conjunction are considered, depending only on the columns present in the block. /// If `expression_ast` is passed, use it to filter block. -void filterBlockWithQuery(const SelectQueryInfo & query_info, Block & block, const Context & context, ASTPtr expression_ast = {}); +void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast = {}); /// Extract from the input stream a set of `name` column values template From 091894f8cac17fffe5dfdd7aa1f088aaf0347439 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 8 Mar 2021 11:09:06 +0800 Subject: [PATCH 021/464] Add more comments --- src/Storages/VirtualColumnUtils.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 537ef91f4f5..4e2c9577089 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -127,10 +127,10 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va bool prepareFilterBlockWithQuery(const ASTPtr & query, const Context & context, Block block, ASTPtr & expression_ast) { - bool ret = true; + bool unmodified = true; const auto & select = query->as(); if (!select.where() && !select.prewhere()) - return ret; + return unmodified; ASTPtr condition_ast; if (select.prewhere() && select.where()) @@ -138,10 +138,11 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, const Context & context, else condition_ast = select.prewhere() ? select.prewhere()->clone() : select.where()->clone(); - // Prepare a block with valid expressions + // Prepare a constant block with valid expressions for (size_t i = 0; i < block.columns(); ++i) block.getByPosition(i).column = block.getByPosition(i).type->createColumnConstWithDefaultValue(1); + // Collect all expression columns in expression_ast. Constant expressions will have constant columns. auto actions = std::make_shared(block.getColumnsWithTypeAndName()); PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; @@ -152,15 +153,15 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, const Context & context, auto expression_actions = std::make_shared(actions); expression_actions->execute(block); - /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. + /// Create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. std::vector functions; if (select.where()) - ret &= extractFunctions(select.where(), block, functions); + unmodified &= extractFunctions(select.where(), block, functions); if (select.prewhere()) - ret &= extractFunctions(select.prewhere(), block, functions); + unmodified &= extractFunctions(select.prewhere(), block, functions); expression_ast = buildWhereExpression(functions); - return ret; + return unmodified; } void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context, ASTPtr expression_ast) From 16f4c02d42dc33f2bfe1bb8c4139ad5d3bd02230 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 7 Mar 2021 22:23:45 +0300 Subject: [PATCH 022/464] Add optimize_skip_unused_shards_limit Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached --- docs/en/operations/settings/settings.md | 8 ++++++++ src/Core/Settings.h | 1 + src/Storages/StorageDistributed.cpp | 5 +++++ ...7_optimize_skip_unused_shards_limit.reference | 0 .../01757_optimize_skip_unused_shards_limit.sql | 16 ++++++++++++++++ 5 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.reference create mode 100644 tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 92587dbcf8f..93f6c193ef3 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1514,6 +1514,14 @@ FORMAT PrettyCompactMonoBlock Default value: 0 +## optimize_skip_unused_shards_limit {#optimize-skip-unused-shards-limit} + +Limit for number of sharding key values, turns off `optimize_skip_unused_shards` if the limit is reached. + +Too many values may require significant amount for processing, while the benefit is doubtful, since if you have huge number of values in `IN (...)`, then most likely the query will be sent to all shards anyway. + +Default value: 1000 + ## optimize_skip_unused_shards {#optimize-skip-unused-shards} Enables or disables skipping of unused shards for [SELECT](../../sql-reference/statements/select/index.md) queries that have sharding key condition in `WHERE/PREWHERE` (assuming that the data is distributed by sharding key, otherwise does nothing). diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 896b56b4d0b..15cbd917d09 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -116,6 +116,7 @@ class IColumn; M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards. If 2 - same as 1 but also apply ORDER BY and LIMIT stages", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ + M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ M(Bool, allow_nondeterministic_optimize_skip_unused_shards, false, "Allow non-deterministic functions (includes dictGet) in sharding_key for optimize_skip_unused_shards", 0) \ M(UInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \ diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3ac6dde836b..e7ab33ef413 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -915,6 +915,11 @@ ClusterPtr StorageDistributed::skipUnusedShards( return nullptr; } + // Too huge number of values + size_t limit = context.getSettingsRef().optimize_skip_unused_shards_limit; + if (limit && blocks->size() > limit) + return nullptr; + std::set shards; for (const auto & block : *blocks) diff --git a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.reference b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql new file mode 100644 index 00000000000..8217b6d1bc4 --- /dev/null +++ b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql @@ -0,0 +1,16 @@ +drop table if exists dist_01757; +create table dist_01757 as system.one engine=Distributed(test_cluster_two_shards, system, one, dummy); + +set optimize_skip_unused_shards=1; +set force_optimize_skip_unused_shards=2; + +select * from dist_01757 where dummy in (0,) format Null; +select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=2; +select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=0; + +select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } +select * from dist_01757 where dummy = 0 or dummy = 1 format Null settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } + +select * from dist_01757 where dummy = 0 and dummy = 1 format Null settings optimize_skip_unused_shards_limit=1; + +drop table dist_01757; From ed09897eb12e49a008c1b68bc0ae2a77c994300b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 10:05:56 +0300 Subject: [PATCH 023/464] Pass optimize_skip_unused_shards_limit to the bottom layer And now optimize_skip_unused_shards_limit=0 is not a special case anymore. --- .../evaluateConstantExpression.cpp | 40 ++++++++++++++----- src/Interpreters/evaluateConstantExpression.h | 3 +- src/Storages/StorageDistributed.cpp | 24 ++++++++--- ...1757_optimize_skip_unused_shards_limit.sql | 25 ++++++++++-- 4 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index db19c000cfd..a3301bcf55b 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -166,9 +166,9 @@ namespace return result; } - Disjunction analyzeFunction(const ASTFunction * fn, const ExpressionActionsPtr & expr) + Disjunction analyzeFunction(const ASTFunction * fn, const ExpressionActionsPtr & expr, size_t & limit) { - if (!fn) + if (!fn || !limit) { return {}; } @@ -182,6 +182,7 @@ namespace const auto * identifier = left->as() ? left->as() : right->as(); const auto * literal = left->as() ? left->as() : right->as(); + --limit; return analyzeEquals(identifier, literal, expr); } else if (fn->name == "in") @@ -192,6 +193,19 @@ namespace Disjunction result; + auto add_dnf = [&](const auto &dnf) + { + if (dnf.size() > limit) + { + result.clear(); + return false; + } + + result.insert(result.end(), dnf.begin(), dnf.end()); + limit -= dnf.size(); + return true; + }; + if (const auto * tuple_func = right->as(); tuple_func && tuple_func->name == "tuple") { const auto * tuple_elements = tuple_func->children.front()->as(); @@ -205,7 +219,10 @@ namespace return {}; } - result.insert(result.end(), dnf.begin(), dnf.end()); + if (!add_dnf(dnf)) + { + return {}; + } } } else if (const auto * tuple_literal = right->as(); @@ -221,7 +238,10 @@ namespace return {}; } - result.insert(result.end(), dnf.begin(), dnf.end()); + if (!add_dnf(dnf)) + { + return {}; + } } } else @@ -244,13 +264,14 @@ namespace for (const auto & arg : args->children) { - const auto dnf = analyzeFunction(arg->as(), expr); + const auto dnf = analyzeFunction(arg->as(), expr, limit); if (dnf.empty()) { return {}; } + /// limit accounted in analyzeFunction() result.insert(result.end(), dnf.begin(), dnf.end()); } @@ -269,13 +290,14 @@ namespace for (const auto & arg : args->children) { - const auto dnf = analyzeFunction(arg->as(), expr); + const auto dnf = analyzeFunction(arg->as(), expr, limit); if (dnf.empty()) { continue; } + /// limit accounted in analyzeFunction() result = andDNF(result, dnf); } @@ -286,15 +308,15 @@ namespace } } -std::optional evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr) +std::optional evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr, size_t & limit) { Blocks result; if (const auto * fn = node->as()) { - const auto dnf = analyzeFunction(fn, target_expr); + const auto dnf = analyzeFunction(fn, target_expr, limit); - if (dnf.empty()) + if (dnf.empty() || !limit) { return {}; } diff --git a/src/Interpreters/evaluateConstantExpression.h b/src/Interpreters/evaluateConstantExpression.h index 8e3fa08a626..c797b8461de 100644 --- a/src/Interpreters/evaluateConstantExpression.h +++ b/src/Interpreters/evaluateConstantExpression.h @@ -46,10 +46,11 @@ ASTPtr evaluateConstantExpressionForDatabaseName(const ASTPtr & node, const Cont /** Try to fold condition to countable set of constant values. * @param node a condition that we try to fold. * @param target_expr expression evaluated over a set of constants. + * @param limit limit for number of values * @return optional blocks each with a single row and a single column for target expression, * or empty blocks if condition is always false, * or nothing if condition can't be folded to a set of constants. */ -std::optional evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr); +std::optional evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr, size_t & limit); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e7ab33ef413..28156458342 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -907,7 +907,24 @@ ClusterPtr StorageDistributed::skipUnusedShards( } replaceConstantExpressions(condition_ast, context, metadata_snapshot->getColumns().getAll(), shared_from_this(), metadata_snapshot); - const auto blocks = evaluateExpressionOverConstantCondition(condition_ast, sharding_key_expr); + + size_t limit = context.getSettingsRef().optimize_skip_unused_shards_limit; + if (!limit || limit > SSIZE_MAX) + { + throw Exception("optimize_skip_unused_shards_limit out of range (0, {}]", ErrorCodes::ARGUMENT_OUT_OF_BOUND, SSIZE_MAX); + } + // To interpret limit==0 as limit is reached + ++limit; + const auto blocks = evaluateExpressionOverConstantCondition(condition_ast, sharding_key_expr, limit); + + if (!limit) + { + LOG_TRACE(log, + "Number of values for sharding key exceeds optimize_skip_unused_shards_limit={}, " + "try to increase it, but note that this may increase query processing time.", + context.getSettingsRef().optimize_skip_unused_shards_limit); + return nullptr; + } // Can't get definite answer if we can skip any shards if (!blocks) @@ -915,11 +932,6 @@ ClusterPtr StorageDistributed::skipUnusedShards( return nullptr; } - // Too huge number of values - size_t limit = context.getSettingsRef().optimize_skip_unused_shards_limit; - if (limit && blocks->size() > limit) - return nullptr; - std::set shards; for (const auto & block : *blocks) diff --git a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql index 8217b6d1bc4..68247dbfbe5 100644 --- a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql +++ b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql @@ -4,13 +4,30 @@ create table dist_01757 as system.one engine=Distributed(test_cluster_two_shards set optimize_skip_unused_shards=1; set force_optimize_skip_unused_shards=2; +-- in select * from dist_01757 where dummy in (0,) format Null; select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=2; -select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=0; -select * from dist_01757 where dummy in (0, 1) format Null settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } -select * from dist_01757 where dummy = 0 or dummy = 1 format Null settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } +-- in negative +select * from dist_01757 where dummy in (0, 1) settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } -select * from dist_01757 where dummy = 0 and dummy = 1 format Null settings optimize_skip_unused_shards_limit=1; +-- or negative +select * from dist_01757 where dummy = 0 or dummy = 1 settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } + +-- or +select * from dist_01757 where dummy = 0 or dummy = 1 format Null settings optimize_skip_unused_shards_limit=2; + +-- and negative +select * from dist_01757 where dummy = 0 and dummy = 1 settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } +select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } +select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=2; -- { serverError 507 } + +-- and +select * from dist_01757 where dummy = 0 and dummy = 1 settings optimize_skip_unused_shards_limit=2; +select * from dist_01757 where dummy = 0 and dummy = 1 and dummy = 3 settings optimize_skip_unused_shards_limit=3; + +-- ARGUMENT_OUT_OF_BOUND error +select * from dist_01757 where dummy in (0, 1) settings optimize_skip_unused_shards_limit=0; -- { serverError 69 } +select * from dist_01757 where dummy in (0, 1) settings optimize_skip_unused_shards_limit=9223372036854775808; -- { serverError 69 } drop table dist_01757; From 20dd448cd70d9dedda31fab7ae2342a1e05fcb4f Mon Sep 17 00:00:00 2001 From: ana-uvarova Date: Mon, 8 Mar 2021 15:16:01 +0300 Subject: [PATCH 024/464] polishing --- docs/en/sql-reference/functions/string-search-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 4415ee42e2e..80f973d3cee 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -15,7 +15,7 @@ The search is case-sensitive by default in all these functions. There are separa Returns the position (in bytes) of the found substring in the string, starting from 1. !!! note - Now it supports position(needle in haystack) syntax for SQL-compatibility, which is same as position(haystack, needle). This new function POSITION(needle IN haystack) works exactly same as POSITION(haystack, needle). + Now it supports position(needle in haystack) syntax for SQL-compatibility, which is same as position(haystack, needle). This new function POSITION(needle IN haystack) works exactly the same way as POSITION(haystack, needle). For a case-insensitive search, use the function [positionCaseInsensitive](#positioncaseinsensitive). From 248478107059a8b6f6c4700b98968db103f0ad51 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 8 Mar 2021 20:26:38 +0300 Subject: [PATCH 025/464] fix MV recovery in Replicated db --- src/Databases/DatabaseReplicated.cpp | 8 ++-- src/Interpreters/InterpreterCreateQuery.cpp | 23 ++++++++--- .../InterpreterShowCreateQuery.cpp | 1 + src/Parsers/ASTCreateQuery.cpp | 8 ++++ src/Parsers/ASTCreateQuery.h | 1 + src/Parsers/ParserCreateQuery.cpp | 14 ++++++- src/Storages/StorageMaterializedView.cpp | 3 +- src/Storages/System/StorageSystemTables.cpp | 1 + .../configs/settings.xml | 1 + .../test_replicated_database/test.py | 39 +++++++++++++------ 10 files changed, 76 insertions(+), 23 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 12cff3407d3..74933b149e6 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -557,12 +557,14 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node auto ast = parseQuery(parser, query, description, 0, global_context.getSettingsRef().max_parser_depth); auto & create = ast->as(); - if (create.uuid == UUIDHelpers::Nil || create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER || ! create.database.empty()) + if (create.uuid == UUIDHelpers::Nil || create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER || !create.database.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected query from {}: {}", node_name, query); + bool is_materialized_view_with_inner_table = create.is_materialized_view && create.to_table_id.empty(); + create.database = getDatabaseName(); create.table = unescapeForFileName(node_name); - create.attach = false; + create.attach = is_materialized_view_with_inner_table; return ast; } @@ -598,7 +600,7 @@ void DatabaseReplicated::shutdown() void DatabaseReplicated::dropTable(const Context & context, const String & table_name, bool no_delay) { auto txn = context.getZooKeeperMetadataTransaction(); - assert(!ddl_worker->isCurrentlyActive() || txn); + assert(!ddl_worker->isCurrentlyActive() || txn || startsWith(table_name, ".inner_id.")); if (txn && txn->isInitialQuery()) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d1af86e7b11..7c7403f678b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -712,6 +712,19 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const } } +static void generateUUIDForTable(ASTCreateQuery & create) +{ + if (create.uuid == UUIDHelpers::Nil) + create.uuid = UUIDHelpers::generateV4(); + + /// If destination table (to_table_id) is not specified for materialized view, + /// then MV will create inner table. We should generate UUID of inner table here, + /// so it will be the same on all hosts if query in ON CLUSTER or database engine is Replicated. + bool need_uuid_for_inner_table = create.is_materialized_view && !create.to_table_id; + if (need_uuid_for_inner_table && create.to_inner_uuid == UUIDHelpers::Nil) + create.to_inner_uuid = UUIDHelpers::generateV4(); +} + void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const { const auto * kind = create.is_dictionary ? "Dictionary" : "Table"; @@ -743,18 +756,19 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data kind_upper, create.table); } - if (create.uuid == UUIDHelpers::Nil) - create.uuid = UUIDHelpers::generateV4(); + generateUUIDForTable(create); } else { bool is_on_cluster = context.getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - if (create.uuid != UUIDHelpers::Nil && !is_on_cluster) + bool has_uuid = create.uuid != UUIDHelpers::Nil || create.to_inner_uuid != UUIDHelpers::Nil; + if (has_uuid && !is_on_cluster) throw Exception(ErrorCodes::INCORRECT_QUERY, "{} UUID specified, but engine of database {} is not Atomic", kind, create.database); /// Ignore UUID if it's ON CLUSTER query create.uuid = UUIDHelpers::Nil; + create.to_inner_uuid = UUIDHelpers::Nil; } if (create.replace_table) @@ -1134,8 +1148,7 @@ void InterpreterCreateQuery::prepareOnClusterQuery(ASTCreateQuery & create, cons /// For CREATE query generate UUID on initiator, so it will be the same on all hosts. /// It will be ignored if database does not support UUIDs. - if (create.uuid == UUIDHelpers::Nil) - create.uuid = UUIDHelpers::generateV4(); + generateUUIDForTable(create); /// For cross-replication cluster we cannot use UUID in replica path. String cluster_name_expanded = context.getMacros()->expand(cluster_name); diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 10c8339c135..a853c4b9a9b 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -82,6 +82,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { auto & create = create_query->as(); create.uuid = UUIDHelpers::Nil; + create.to_inner_uuid = UUIDHelpers::Nil; } WriteBufferFromOwnString buf; diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 2af0d2d4a45..1192fcc6ebd 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -297,12 +297,20 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (to_table_id) { + assert(is_materialized_view && to_inner_uuid == UUIDHelpers::Nil); settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : "") << (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "") << backQuoteIfNeed(to_table_id.table_name); } + if (to_inner_uuid != UUIDHelpers::Nil) + { + assert(is_materialized_view && !to_table_id); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO INNER UUID " << (settings.hilite ? hilite_none : "") + << quoteString(toString(to_inner_uuid)); + } + if (!as_table.empty()) { settings.ostr diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index c9a6251cb94..d6d5c22240c 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -66,6 +66,7 @@ public: ASTExpressionList * tables = nullptr; StorageID to_table_id = StorageID::createEmpty(); /// For CREATE MATERIALIZED VIEW mv TO table. + UUID to_inner_uuid = UUIDHelpers::Nil; /// For materialized view with inner table ASTStorage * storage = nullptr; String as_database; String as_table; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 4cef79fdf42..bfd51b7633d 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -780,6 +780,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ASTPtr table; ASTPtr to_table; + ASTPtr to_inner_uuid; ASTPtr columns_list; ASTPtr storage; ASTPtr as_database; @@ -830,9 +831,16 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec return false; } - // TO [db.]table - if (ParserKeyword{"TO"}.ignore(pos, expected)) + + if (ParserKeyword{"TO INNER UUID"}.ignore(pos, expected)) { + ParserLiteral literal_p; + if (!literal_p.parse(pos, to_inner_uuid, expected)) + return false; + } + else if (ParserKeyword{"TO"}.ignore(pos, expected)) + { + // TO [db.]table if (!table_name_p.parse(pos, to_table, expected)) return false; } @@ -883,6 +891,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (to_table) query->to_table_id = getTableIdentifier(to_table); + if (to_inner_uuid) + query->to_inner_uuid = parseFromString(to_inner_uuid->as()->value.get()); query->set(query->columns_list, columns_list); query->set(query->storage, storage); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d2cb418ce97..f915f76bbfc 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -84,7 +84,7 @@ StorageMaterializedView::StorageMaterializedView( else if (attach_) { /// If there is an ATTACH request, then the internal table must already be created. - target_table_id = StorageID(getStorageID().database_name, generateInnerTableName(getStorageID())); + target_table_id = StorageID(getStorageID().database_name, generateInnerTableName(getStorageID()), query.to_inner_uuid); } else { @@ -93,6 +93,7 @@ StorageMaterializedView::StorageMaterializedView( auto manual_create_query = std::make_shared(); manual_create_query->database = getStorageID().database_name; manual_create_query->table = generateInnerTableName(getStorageID()); + manual_create_query->uuid = query.to_inner_uuid; auto new_columns_list = std::make_shared(); new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr()); diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 132ed234323..fffe909715f 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -359,6 +359,7 @@ protected: { auto & create = ast->as(); create.uuid = UUIDHelpers::Nil; + create.to_inner_uuid = UUIDHelpers::Nil; } if (columns_mask[src_index++]) diff --git a/tests/integration/test_replicated_database/configs/settings.xml b/tests/integration/test_replicated_database/configs/settings.xml index e0f7e8691e6..7f45502e20d 100644 --- a/tests/integration/test_replicated_database/configs/settings.xml +++ b/tests/integration/test_replicated_database/configs/settings.xml @@ -2,6 +2,7 @@ 1 + 1 diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 99e7d6077f8..487f2b5a8c1 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -198,8 +198,14 @@ def test_recover_staled_replica(started_cluster): dummy_node.query("CREATE TABLE recover.rmt2 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) main_node.query("CREATE TABLE recover.rmt3 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) dummy_node.query("CREATE TABLE recover.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) - main_node.query("CREATE DICTIONARY recover.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") - dummy_node.query("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") + main_node.query("CREATE MATERIALIZED VIEW recover.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", settings=settings) + dummy_node.query("CREATE MATERIALIZED VIEW recover.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", settings=settings) + main_node.query("CREATE DICTIONARY recover.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") + dummy_node.query("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") for table in ['t1', 't2', 'mt1', 'mt2', 'rmt1', 'rmt2', 'rmt3', 'rmt5']: main_node.query("INSERT INTO recover.{} VALUES (42)".format(table)) @@ -217,35 +223,44 @@ def test_recover_staled_replica(started_cluster): main_node.query("RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings) main_node.query("DROP TABLE recover.rmt5", settings=settings) main_node.query("DROP DICTIONARY recover.d2", settings=settings) - main_node.query("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", settings=settings) + main_node.query("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", settings=settings) + + inner_table = ".inner_id." + dummy_node.query("SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'").strip() + main_node.query("ALTER TABLE recover.`{}` MODIFY COLUMN n int DEFAULT 42".format(inner_table), settings=settings) + main_node.query("ALTER TABLE recover.mv1 MODIFY QUERY SELECT m FROM recover.rmt1".format(inner_table), settings=settings) + main_node.query("RENAME TABLE recover.mv2 TO recover.mv3".format(inner_table), settings=settings) main_node.query("CREATE TABLE recover.tmp AS recover.m1", settings=settings) main_node.query("DROP TABLE recover.tmp", settings=settings) main_node.query("CREATE TABLE recover.tmp AS recover.m1", settings=settings) main_node.query("DROP TABLE recover.tmp", settings=settings) main_node.query("CREATE TABLE recover.tmp AS recover.m1", settings=settings) - main_node.query("DROP TABLE recover.tmp", settings=settings) - main_node.query("CREATE TABLE recover.tmp AS recover.m1", settings=settings) - assert main_node.query("SELECT name FROM system.tables WHERE database='recover' ORDER BY name") == "d1\nd2\nm1\nmt1\nmt2\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" - query = "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' ORDER BY name" + assert main_node.query("SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name") == \ + "d1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" + query = "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " \ + "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" expected = main_node.query(query) assert_eq_with_retry(dummy_node, query, expected) + assert main_node.query("SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'") == "2\n" + assert dummy_node.query("SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'") == "2\n" - for table in ['m1', 't2', 'mt1', 'mt2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2']: + for table in ['m1', 't2', 'mt1', 'mt2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2', 'mv1', 'mv3']: assert main_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" - for table in ['t2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2', 'mt2']: + for table in ['t2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2', 'mt2', 'mv1', 'mv3']: assert dummy_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" for table in ['m1', 'mt1']: assert dummy_node.query("SELECT count() FROM recover.{}".format(table)) == "0\n" assert dummy_node.query("SELECT count() FROM system.tables WHERE database='recover_broken_tables'") == "2\n" - table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'mt1_26_%'").strip() + table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'mt1_29_%'").strip() assert dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) == "42\n" - table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'rmt5_26_%'").strip() + table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'rmt5_29_%'").strip() assert dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) == "42\n" - expected = "Cleaned 4 outdated objects: dropped 1 dictionaries and 1 tables, moved 2 tables" + expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" assert_logs_contain(dummy_node, expected) dummy_node.query("DROP TABLE recover.tmp") From 2022b90919d1c183d5212f43835e1f81f5821d8b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 8 Mar 2021 23:35:09 +0300 Subject: [PATCH 026/464] configurable output mode for distributed DDL --- src/Core/Settings.h | 3 +- src/Core/SettingsEnums.cpp | 6 + src/Core/SettingsEnums.h | 11 ++ src/Databases/DatabaseReplicated.cpp | 9 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 118 ++++++++++++++---- src/Interpreters/executeDDLQueryOnCluster.h | 5 + tests/config/users.d/database_replicated.xml | 2 +- .../test_replicated_database/test.py | 10 +- ...1175_distributed_ddl_output_mode.reference | 25 ++++ .../01175_distributed_ddl_output_mode.sh | 39 ++++++ 10 files changed, 190 insertions(+), 38 deletions(-) create mode 100644 tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference create mode 100755 tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8afc08da21a..8573b3be303 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -434,7 +434,7 @@ class IColumn; M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ M(Bool, allow_experimental_database_replicated, false, "Allow to create databases with Replicated engine", 0) \ M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ - M(Bool, database_replicated_ddl_output, true, "Return table with query execution status as a result of DDL query", 0) \ + M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ @@ -446,6 +446,7 @@ class IColumn; M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ + M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing. Will be removed after 2021-09-08", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 2e1cf025256..64ba51d1c68 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -102,4 +102,10 @@ IMPLEMENT_SETTING_ENUM(UnionMode, ErrorCodes::UNKNOWN_UNION, {"ALL", UnionMode::ALL}, {"DISTINCT", UnionMode::DISTINCT}}) +IMPLEMENT_SETTING_ENUM(DistributedDDLOutputMode, ErrorCodes::BAD_ARGUMENTS, + {{"none", DistributedDDLOutputMode::NONE}, + {"throw", DistributedDDLOutputMode::THROW}, + {"null_status_on_timeout", DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT}, + {"never_throw", DistributedDDLOutputMode::NEVER_THROW}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index c2ef08135eb..7615b185a61 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -138,4 +138,15 @@ enum class UnionMode DECLARE_SETTING_ENUM(UnionMode) + +enum class DistributedDDLOutputMode +{ + NONE, + THROW, + NULL_STATUS_ON_TIMEOUT, + NEVER_THROW, +}; + +DECLARE_SETTING_ENUM(DistributedDDLOutputMode) + } diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 74933b149e6..fefe3f1b6b7 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -313,15 +313,8 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const entry.initiator = ddl_worker->getCommonHostID(); String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context); - BlockIO io; - if (query_context.getSettingsRef().distributed_ddl_task_timeout == 0) - return io; - Strings hosts_to_wait = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - auto stream = std::make_shared(node_path, entry, query_context, hosts_to_wait); - if (query_context.getSettingsRef().database_replicated_ddl_output) - io.in = std::move(stream); - return io; + return getDistributedDDLStatus(node_path, entry, query_context, hosts_to_wait); } static UUID getTableUUIDIfReplicated(const String & metadata, const Context & context) diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 1937fbaf905..2bd69c683f1 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -13,6 +13,9 @@ #include #include #include +#include +#include +#include #include namespace fs = std::filesystem; @@ -165,16 +168,29 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont entry.initiator = ddl_worker.getCommonHostID(); String node_path = ddl_worker.enqueueQuery(entry); + return getDistributedDDLStatus(node_path, entry, context); +} + +BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, const Context & context, const std::optional & hosts_to_wait) +{ BlockIO io; if (context.getSettingsRef().distributed_ddl_task_timeout == 0) return io; - auto stream = std::make_shared(node_path, entry, context); - io.in = std::move(stream); + auto stream = std::make_shared(node_path, entry, context, hosts_to_wait); + if (context.getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NONE) + { + /// Wait for query to finish, but ignore output + NullBlockOutputStream output{Block{}}; + copyData(*stream, output); + } + else + { + io.in = std::move(stream); + } return io; } - DDLQueryStatusInputStream::DDLQueryStatusInputStream(const String & zk_node_path, const DDLLogEntry & entry, const Context & context_, const std::optional & hosts_to_wait) : node_path(zk_node_path) @@ -182,13 +198,29 @@ DDLQueryStatusInputStream::DDLQueryStatusInputStream(const String & zk_node_path , watch(CLOCK_MONOTONIC_COARSE) , log(&Poco::Logger::get("DDLQueryStatusInputStream")) { + if (context.getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::THROW || + context.getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NONE) + throw_on_timeout = true; + else if (context.getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT || + context.getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NEVER_THROW) + throw_on_timeout = false; + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown output mode"); + + auto maybe_make_nullable = [&](const DataTypePtr & type) -> DataTypePtr + { + if (throw_on_timeout) + return type; + return std::make_shared(type); + }; + sample = Block{ - {std::make_shared(), "host"}, - {std::make_shared(), "port"}, - {std::make_shared(), "status"}, - {std::make_shared(), "error"}, - {std::make_shared(), "num_hosts_remaining"}, - {std::make_shared(), "num_hosts_active"}, + {std::make_shared(), "host"}, + {std::make_shared(), "port"}, + {maybe_make_nullable(std::make_shared()), "status"}, + {maybe_make_nullable(std::make_shared()), "error"}, + {std::make_shared(), "num_hosts_remaining"}, + {std::make_shared(), "num_hosts_active"}, }; if (hosts_to_wait) @@ -205,14 +237,33 @@ DDLQueryStatusInputStream::DDLQueryStatusInputStream(const String & zk_node_path addTotalRowsApprox(waiting_hosts.size()); timeout_seconds = context.getSettingsRef().distributed_ddl_task_timeout; + /// There is not sense to check query status with zero timeout. + assert(timeout_seconds >= 0); +} + +std::pair DDLQueryStatusInputStream::parseHostAndPort(const String & host_id) const +{ + String host = host_id; + UInt16 port = 0; + if (by_hostname) + { + auto host_and_port = Cluster::Address::fromString(host_id); + host = host_and_port.first; + port = host_and_port.second; + } + return {host, port}; } Block DDLQueryStatusInputStream::readImpl() { Block res; - if (num_hosts_finished >= waiting_hosts.size()) + bool all_hosts_finished = num_hosts_finished >= waiting_hosts.size(); + /// Seems like num_hosts_finished cannot be strictly greater than waiting_hosts.size() + assert(num_hosts_finished <= waiting_hosts.size()); + if (all_hosts_finished || timeout_exceeded) { - if (first_exception) + bool throw_if_error_on_host = context.getSettingsRef().distributed_ddl_output_mode != DistributedDDLOutputMode::NEVER_THROW; + if (first_exception && throw_if_error_on_host) throw Exception(*first_exception); return res; @@ -225,22 +276,46 @@ Block DDLQueryStatusInputStream::readImpl() { if (isCancelled()) { - if (first_exception) + bool throw_if_error_on_host = context.getSettingsRef().distributed_ddl_output_mode != DistributedDDLOutputMode::NEVER_THROW; + if (first_exception && throw_if_error_on_host) throw Exception(*first_exception); return res; } - if (timeout_seconds >= 0 && watch.elapsedSeconds() > timeout_seconds) + if (watch.elapsedSeconds() > timeout_seconds) { size_t num_unfinished_hosts = waiting_hosts.size() - num_hosts_finished; size_t num_active_hosts = current_active_hosts.size(); + constexpr const char * msg_format = "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " + "There are {} unfinished hosts ({} of them are currently active), " + "they are going to execute the query in background"; + if (throw_on_timeout) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, msg_format, + node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, - "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " - "There are {} unfinished hosts ({} of them are currently active), they are going to execute the query in background", - node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); + timeout_exceeded = true; + LOG_INFO(log, msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); + + NameSet unfinished_hosts = waiting_hosts; + for (const auto & host_id : finished_hosts) + unfinished_hosts.erase(host_id); + + /// Query is not finished on the rest hosts, so fill the corresponding rows with NULLs. + MutableColumns columns = sample.cloneEmptyColumns(); + for (const String & host_id : unfinished_hosts) + { + auto [host, port] = parseHostAndPort(host_id); + columns[0]->insert(host); + columns[1]->insert(port); + columns[2]->insert(Field{}); + columns[3]->insert(Field{}); + columns[4]->insert(num_unfinished_hosts); + columns[5]->insert(num_active_hosts); + } + res = sample.cloneWithColumns(std::move(columns)); + return res; } if (num_hosts_finished != 0 || try_number != 0) @@ -272,14 +347,7 @@ Block DDLQueryStatusInputStream::readImpl() status.tryDeserializeText(status_data); } - String host = host_id; - UInt16 port = 0; - if (by_hostname) - { - auto host_and_port = Cluster::Address::fromString(host_id); - host = host_and_port.first; - port = host_and_port.second; - } + auto [host, port] = parseHostAndPort(host_id); if (status.code != 0 && first_exception == nullptr) first_exception = std::make_unique(status.code, "There was an error on [{}:{}]: {}", host, port, status.message); diff --git a/src/Interpreters/executeDDLQueryOnCluster.h b/src/Interpreters/executeDDLQueryOnCluster.h index 2b272d3b0da..196cd7c5c2c 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.h +++ b/src/Interpreters/executeDDLQueryOnCluster.h @@ -24,6 +24,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & conte BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option = false); BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option = false); +BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, const Context & context, const std::optional & hosts_to_wait = {}); class DDLQueryStatusInputStream final : public IBlockInputStream { @@ -44,6 +45,8 @@ private: Strings getNewAndUpdate(const Strings & current_list_of_finished_hosts); + std::pair parseHostAndPort(const String & host_id) const; + String node_path; const Context & context; Stopwatch watch; @@ -62,6 +65,8 @@ private: Int64 timeout_seconds = 120; bool by_hostname = true; + bool throw_on_timeout = true; + bool timeout_exceeded = false; }; } diff --git a/tests/config/users.d/database_replicated.xml b/tests/config/users.d/database_replicated.xml index 23801d00154..27408331517 100644 --- a/tests/config/users.d/database_replicated.xml +++ b/tests/config/users.d/database_replicated.xml @@ -2,7 +2,7 @@ 1 - 0 + none 30 30 diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 487f2b5a8c1..13f38f57101 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -103,12 +103,16 @@ def test_alters_from_different_replicas(started_cluster): dummy_node.stop_clickhouse(kill=True) - settings = {"distributed_ddl_task_timeout": 10} + settings = {"distributed_ddl_task_timeout": 5} assert "There are 1 unfinished hosts (0 of them are currently active)" in \ competing_node.query_and_get_error("ALTER TABLE testdb.concurrent_test ADD COLUMN Added0 UInt32;", settings=settings) + settings = {"distributed_ddl_task_timeout": 5, "distributed_ddl_output_mode": "null_status_on_timeout"} + assert "shard1|replica2\t0\t\\N\t\\N" in \ + main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added2 UInt32;", settings=settings) + settings = {"distributed_ddl_task_timeout": 5, "distributed_ddl_output_mode": "never_throw"} + assert "shard1|replica2\t0\t\\N\t\\N" in \ + competing_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", settings=settings) dummy_node.start_clickhouse() - main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added2 UInt32;") - competing_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;") main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;") competing_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;") main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;") diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference b/tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference new file mode 100644 index 00000000000..4eb8143580b --- /dev/null +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference @@ -0,0 +1,25 @@ +none +Received exception from server: +Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57, e.displayText() = Error: Table default.throw already exists +Received exception from server: +Code: 159. Error: Received from localhost:9000. Error: Watching task is executing longer than distributed_ddl_task_timeout (=5) seconds. There are 1 unfinished hosts (0 of them are currently active), they are going to execute the query in background. +throw +localhost 9000 0 0 0 +localhost 9000 57 Code: 57, e.displayText() = Error: Table default.throw already exists. 0 0 +Received exception from server: +Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57, e.displayText() = Error: Table default.throw already exists +localhost 9000 0 1 0 +Received exception from server: +Code: 159. Error: Received from localhost:9000. Error: Watching task is executing longer than distributed_ddl_task_timeout (=5) seconds. There are 1 unfinished hosts (0 of them are currently active), they are going to execute the query in background. +null_status_on_timeout +localhost 9000 0 0 0 +localhost 9000 57 Code: 57, e.displayText() = Error: Table default.null_status already exists. 0 0 +Received exception from server: +Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57, e.displayText() = Error: Table default.null_status already exists +localhost 9000 0 1 0 +localhost 1 \N \N 1 0 +never_throw +localhost 9000 0 0 0 +localhost 9000 57 Code: 57, e.displayText() = Error: Table default.never_throw already exists. 0 0 +localhost 9000 0 1 0 +localhost 1 \N \N 1 0 diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh new file mode 100755 index 00000000000..f8b246f4dd3 --- /dev/null +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "drop table if exists throw;" +$CLICKHOUSE_CLIENT -q "drop table if exists null_status;" +$CLICKHOUSE_CLIENT -q "drop table if exists never_throw;" + +CLICKHOUSE_CLIENT_OPT=$(echo ${CLICKHOUSE_CLIENT_OPT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=fatal/g') + +CLIENT="$CLICKHOUSE_CLIENT_BINARY $CLICKHOUSE_CLIENT_OPT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=none" +$CLIENT -q "select value from system.settings where name='distributed_ddl_output_mode';" +# Ok +$CLIENT -q "create table throw on cluster test_shard_localhost (n int) engine=Memory;" +# Table exists +$CLIENT -q "create table throw on cluster test_shard_localhost (n int) engine=Memory;" 2>&1| grep -Fv "@ 0x" | sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" | sed "s/exists.. /exists/" +# Timeout +$CLIENT -q "drop table throw on cluster test_unavailable_shard;" 2>&1| grep -Fv "@ 0x" | sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" | sed "s/Watching task .* is executing longer/Watching task is executing longer/" | sed "s/background. /background./" + +CLIENT="$CLICKHOUSE_CLIENT_BINARY $CLICKHOUSE_CLIENT_OPT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=throw" +$CLIENT -q "select value from system.settings where name='distributed_ddl_output_mode';" +$CLIENT -q "create table throw on cluster test_shard_localhost (n int) engine=Memory;" +$CLIENT -q "create table throw on cluster test_shard_localhost (n int) engine=Memory;" 2>&1| grep -Fv "@ 0x" | sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" | sed "s/exists.. /exists/" +$CLIENT -q "drop table throw on cluster test_unavailable_shard;" 2>&1| grep -Fv "@ 0x" | sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" | sed "s/Watching task .* is executing longer/Watching task is executing longer/" | sed "s/background. /background./" + +CLIENT="$CLICKHOUSE_CLIENT_BINARY $CLICKHOUSE_CLIENT_OPT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=null_status_on_timeout" +$CLIENT -q "select value from system.settings where name='distributed_ddl_output_mode';" +$CLIENT -q "create table null_status on cluster test_shard_localhost (n int) engine=Memory;" +$CLIENT -q "create table null_status on cluster test_shard_localhost (n int) engine=Memory;" 2>&1| grep -Fv "@ 0x" | sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" | sed "s/exists.. /exists/" +$CLIENT -q "drop table null_status on cluster test_unavailable_shard;" + +CLIENT="$CLICKHOUSE_CLIENT_BINARY $CLICKHOUSE_CLIENT_OPT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=never_throw" +$CLIENT -q "select value from system.settings where name='distributed_ddl_output_mode';" +$CLIENT -q "create table never_throw on cluster test_shard_localhost (n int) engine=Memory;" +$CLIENT -q "create table never_throw on cluster test_shard_localhost (n int) engine=Memory;" 2>&1| sed "s/DB::Exception/Error/g" | sed "s/ (version.*)//" +$CLIENT -q "drop table never_throw on cluster test_unavailable_shard;" From 5070b5b85b78dadd79769324283b02fac58ec729 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 9 Mar 2021 00:34:52 +0300 Subject: [PATCH 027/464] detach permanently setting for Replicated db --- src/Core/Settings.h | 1 + src/Databases/DatabaseReplicated.cpp | 15 +++++++++++++++ src/Interpreters/InterpreterCreateQuery.cpp | 12 +++++++++++- src/Interpreters/InterpreterDropQuery.cpp | 4 ---- tests/config/users.d/database_replicated.xml | 1 + 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8573b3be303..29ba289dc47 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -434,6 +434,7 @@ class IColumn; M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ M(Bool, allow_experimental_database_replicated, false, "Allow to create databases with Replicated engine", 0) \ M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ + M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index fefe3f1b6b7..76f2c798237 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -294,7 +295,11 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const /// Replicas will set correct name of current database in query context (database name can be different on replicas) if (auto * ddl_query = query->as()) + { + if (ddl_query->database != getDatabaseName()) + throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed"); ddl_query->database.clear(); + } if (const auto * query_alter = query->as()) { @@ -305,6 +310,16 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const } } + if (auto * query_drop = query->as()) + { + if (query_drop->kind == ASTDropQuery::Kind::Detach && query_context.getSettingsRef().database_replicated_always_detach_permanently) + query_drop->permanently = true; + if (query_drop->kind == ASTDropQuery::Kind::Detach && !query_drop->permanently) + throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH TABLE is not allowed for Replicated databases. " + "Use DETACH TABLE PERMANENTLY or SYSTEM RESTART REPLICA or set " + "database_replicated_always_detach_permanently to 1"); + } + LOG_DEBUG(log, "Proposing query: {}", queryToString(query)); /// TODO maybe write current settings to log entry? diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7c7403f678b..1c63dee1de2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -817,6 +817,17 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.attach && !create.storage && !create.columns_list) { auto database = DatabaseCatalog::instance().getDatabase(database_name); + if (database->getEngineName() == "Replicated") + { + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.table); + if (typeid_cast(database.get()) && context.getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) + { + create.database = database_name; + guard->releaseTableLock(); + return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, context); + } + } + bool if_not_exists = create.if_not_exists; // Table SQL definition is available even if the table is detached (even permanently) @@ -890,7 +901,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database && database->getEngineName() == "Replicated") { auto guard = DatabaseCatalog::instance().getDDLGuard(create.database, create.table); - database = DatabaseCatalog::instance().getDatabase(create.database); if (typeid_cast(database.get()) && context.getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) { assertOrSetUUID(create, database); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 33e93a79c41..b30996b1dbf 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -133,10 +133,6 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, Dat !is_drop_or_detach_database; if (is_replicated_ddl_query) { - if (query.kind == ASTDropQuery::Kind::Detach && !query.permanently) - throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH TABLE is not allowed for Replicated databases. " - "Use DETACH TABLE PERMANENTLY or SYSTEM RESTART REPLICA"); - if (query.kind == ASTDropQuery::Kind::Detach) context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); else if (query.kind == ASTDropQuery::Kind::Truncate) diff --git a/tests/config/users.d/database_replicated.xml b/tests/config/users.d/database_replicated.xml index 27408331517..f21e3952d99 100644 --- a/tests/config/users.d/database_replicated.xml +++ b/tests/config/users.d/database_replicated.xml @@ -5,6 +5,7 @@ none 30 30 + 1 From 163e27b299a1bc4a22bccbef80b08d01be30b897 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 9 Mar 2021 01:57:53 +0300 Subject: [PATCH 028/464] support query and session settings for distributed DDL --- src/Core/Settings.h | 1 + src/Databases/DatabaseReplicated.cpp | 2 +- src/Interpreters/DDLTask.cpp | 83 +++++++++++++++---- src/Interpreters/DDLTask.h | 7 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 1 + tests/config/users.d/database_replicated.xml | 1 + 6 files changed, 75 insertions(+), 20 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 29ba289dc47..32f1b985ac3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -436,6 +436,7 @@ class IColumn; M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result", 0) \ + M(UInt64, distributed_ddl_entry_format_version, 1, "Version of DDL entry to write into ZooKeeper", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 76f2c798237..612b7fb1948 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -322,10 +322,10 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const LOG_DEBUG(log, "Proposing query: {}", queryToString(query)); - /// TODO maybe write current settings to log entry? DDLLogEntry entry; entry.query = queryToString(query); entry.initiator = ddl_worker->getCommonHostID(); + entry.setSettingsIfRequired(query_context); String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context); Strings hosts_to_wait = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 0c4c8a1bc34..5d3f538f420 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -43,20 +44,47 @@ bool HostID::isLocalAddress(UInt16 clickhouse_port) const } } +void DDLLogEntry::assertVersion() const +{ + constexpr UInt64 MAX_VERSION = 2; + if (version == 0 || MAX_VERSION < version) + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}." + "Maximum supported version is {}", version, MAX_VERSION); +} + +void DDLLogEntry::setSettingsIfRequired(const Context & context) +{ + version = context.getSettingsRef().distributed_ddl_entry_format_version; + if (version == 2) + settings.emplace(context.getSettingsRef().changes()); +} String DDLLogEntry::toString() const { WriteBufferFromOwnString wb; - Strings host_id_strings(hosts.size()); - std::transform(hosts.begin(), hosts.end(), host_id_strings.begin(), HostID::applyToString); - - auto version = CURRENT_VERSION; wb << "version: " << version << "\n"; wb << "query: " << escape << query << "\n"; - wb << "hosts: " << host_id_strings << "\n"; + + bool write_hosts = version == 1 || !hosts.empty(); + if (write_hosts) + { + Strings host_id_strings(hosts.size()); + std::transform(hosts.begin(), hosts.end(), host_id_strings.begin(), HostID::applyToString); + wb << "hosts: " << host_id_strings << "\n"; + } + wb << "initiator: " << initiator << "\n"; + bool write_settings = 1 <= version && settings && !settings->empty(); + if (write_settings) + { + ASTSetQuery ast; + ast.is_standalone = false; + ast.changes = *settings; + wb << "settings: " << serializeAST(ast) << "\n"; + } + return wb.str(); } @@ -64,25 +92,46 @@ void DDLLogEntry::parse(const String & data) { ReadBufferFromString rb(data); - int version; rb >> "version: " >> version >> "\n"; - - if (version != CURRENT_VERSION) - throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}", version); + assertVersion(); Strings host_id_strings; rb >> "query: " >> escape >> query >> "\n"; - rb >> "hosts: " >> host_id_strings >> "\n"; + if (version == 1) + { + rb >> "hosts: " >> host_id_strings >> "\n"; - if (!rb.eof()) - rb >> "initiator: " >> initiator >> "\n"; - else - initiator.clear(); + if (!rb.eof()) + rb >> "initiator: " >> initiator >> "\n"; + else + initiator.clear(); + } + else if (version == 2) + { + + if (!rb.eof() && *rb.position() == 'h') + rb >> "hosts: " >> host_id_strings >> "\n"; + if (!rb.eof() && *rb.position() == 'i') + rb >> "initiator: " >> initiator >> "\n"; + if (!rb.eof() && *rb.position() == 's') + { + String settings_str; + rb >> "settings: " >> settings_str >> "\n"; + ParserSetQuery parser{true}; + constexpr UInt64 max_size = 4096; + constexpr UInt64 max_depth = 16; + ASTPtr settings_ast = parseQuery(parser, settings_str, max_size, max_depth); + settings.emplace(std::move(settings_ast->as()->changes)); + } + } assertEOF(rb); - hosts.resize(host_id_strings.size()); - std::transform(host_id_strings.begin(), host_id_strings.end(), hosts.begin(), HostID::fromString); + if (!host_id_strings.empty()) + { + hosts.resize(host_id_strings.size()); + std::transform(host_id_strings.begin(), host_id_strings.end(), hosts.begin(), HostID::fromString); + } } @@ -102,6 +151,8 @@ std::unique_ptr DDLTaskBase::makeQueryContext(Context & from_context, c query_context->makeQueryContext(); query_context->setCurrentQueryId(""); // generate random query_id query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; + if (entry.settings) + query_context->applySettingsChanges(*entry.settings); return query_context; } diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 45702599fcf..777ab4ecb18 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -56,15 +56,16 @@ struct HostID struct DDLLogEntry { + UInt64 version = 1; String query; std::vector hosts; String initiator; // optional + std::optional settings; - static constexpr int CURRENT_VERSION = 1; - + void setSettingsIfRequired(const Context & context); String toString() const; - void parse(const String & data); + void assertVersion() const; }; struct DDLTaskBase diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 2bd69c683f1..56eab86de3b 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -166,6 +166,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont entry.hosts = std::move(hosts); entry.query = queryToString(query_ptr); entry.initiator = ddl_worker.getCommonHostID(); + entry.setSettingsIfRequired(context); String node_path = ddl_worker.enqueueQuery(entry); return getDistributedDDLStatus(node_path, entry, context); diff --git a/tests/config/users.d/database_replicated.xml b/tests/config/users.d/database_replicated.xml index f21e3952d99..903b8a64e22 100644 --- a/tests/config/users.d/database_replicated.xml +++ b/tests/config/users.d/database_replicated.xml @@ -6,6 +6,7 @@ 30 30 1 + 2 From 5c5d11cfce37c3415f8e4e12eeb77c3789cf2149 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 9 Mar 2021 03:48:37 +0300 Subject: [PATCH 029/464] enable some tests --- src/Databases/DatabaseOnDisk.cpp | 6 +- src/Databases/DatabaseOnDisk.h | 5 +- src/Databases/DatabaseReplicated.cpp | 16 +++ src/Databases/DatabaseReplicated.h | 1 + tests/config/config.d/clusters.xml | 2 +- tests/queries/0_stateless/01666_blns.sql | 4 +- tests/queries/skip_list.json | 166 ++++------------------- 7 files changed, 50 insertions(+), 150 deletions(-) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index e5d2b23ace0..50a7ce4df04 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -231,8 +231,8 @@ void DatabaseOnDisk::createTable( if (create.attach_short_syntax) { /// Metadata already exists, table was detached + removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); attachTable(table_name, table, getTableDataPath(create)); - removeDetachedPermanentlyFlag(table_name, table_metadata_path); return; } @@ -270,12 +270,12 @@ void DatabaseOnDisk::createTable( commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path, context); - removeDetachedPermanentlyFlag(table_name, table_metadata_path); + removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); } /// If the table was detached permanently we will have a flag file with /// .sql.detached extension, is not needed anymore since we attached the table back -void DatabaseOnDisk::removeDetachedPermanentlyFlag(const String & table_name, const String & table_metadata_path) const +void DatabaseOnDisk::removeDetachedPermanentlyFlag(const Context &, const String & table_name, const String & table_metadata_path) const { try { diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index fefe6e91606..8284cd92fdb 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -94,11 +94,10 @@ protected: virtual void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, const String & table_metadata_tmp_path, const String & table_metadata_path, const Context & query_context); + virtual void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const; + const String metadata_path; const String data_path; - -private: - void removeDetachedPermanentlyFlag(const String & table_name, const String & table_metadata_path) const; }; } diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 612b7fb1948..f075361ef7a 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -712,12 +712,28 @@ void DatabaseReplicated::detachTablePermanently(const Context & context, const S assert(!ddl_worker->isCurrentlyActive() || txn); if (txn && txn->isInitialQuery()) { + /// We have to remove metadata from zookeeper, because we do not distinguish permanently detached tables + /// from attached tables when recovering replica. String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); txn->addOp(zkutil::makeRemoveRequest(metadata_zk_path, -1)); } DatabaseAtomic::detachTablePermanently(context, table_name); } +void DatabaseReplicated::removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const +{ + auto txn = context.getZooKeeperMetadataTransaction(); + assert(!ddl_worker->isCurrentlyActive() || txn); + if (txn && txn->isInitialQuery() && !txn->isExecuted()) + { + String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); + String statement = readMetadataFile(table_name); + txn->addOp(zkutil::makeCreateRequest(metadata_zk_path, statement, zkutil::CreateMode::Persistent)); + } + DatabaseAtomic::removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); +} + + String DatabaseReplicated::readMetadataFile(const String & table_name) const { String statement; diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index fde53cf2c29..5290ea39ff6 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -45,6 +45,7 @@ public: const ASTPtr & query) override; void removeDictionary(const Context & context, const String & dictionary_name) override; void detachTablePermanently(const Context & context, const String & table_name) override; + void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const override; /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index c0babf0ff89..b783372f83b 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -17,4 +17,4 @@ - \ No newline at end of file + diff --git a/tests/queries/0_stateless/01666_blns.sql b/tests/queries/0_stateless/01666_blns.sql index 787f0369244..be9632092bc 100644 --- a/tests/queries/0_stateless/01666_blns.sql +++ b/tests/queries/0_stateless/01666_blns.sql @@ -554,9 +554,9 @@ SELECT count() FROM test; DROP TABLE IF EXISTS test_r1; DROP TABLE IF EXISTS test_r2; -CREATE TABLE test_r1 AS test ENGINE = ReplicatedMergeTree('/clickhouse/test', 'r1') ORDER BY "\\" SETTINGS min_bytes_for_wide_part = '100G'; +CREATE TABLE test_r1 AS test ENGINE = ReplicatedMergeTree('/clickhouse/test_01666', 'r1') ORDER BY "\\" SETTINGS min_bytes_for_wide_part = '100G'; INSERT INTO test_r1 SELECT * FROM test; -CREATE TABLE test_r2 AS test ENGINE = ReplicatedMergeTree('/clickhouse/test', 'r2') ORDER BY "\\" SETTINGS min_bytes_for_wide_part = '100G'; +CREATE TABLE test_r2 AS test ENGINE = ReplicatedMergeTree('/clickhouse/test_01666', 'r2') ORDER BY "\\" SETTINGS min_bytes_for_wide_part = '100G'; SYSTEM SYNC REPLICA test_r2; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 45d569fc131..81e26675a1d 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -106,163 +106,47 @@ "00738_lock_for_inner_table" ], "database-replicated": [ - /// Tests with DETACH TABLE (it's not allowed) - /// and tests with SET (session and query settings are not supported) "memory_tracking", "memory_usage", "live_view", - "01413_alter_update_supertype", - "01149_zookeeper_mutation_stuck_after_replace_partition", - "00836_indices_alter_replicated_zookeeper", - "00652_mutations_alter_update", - "01715_tuple_insert_null_as_default", - "00825_protobuf_format_map", - "00152_insert_different_granularity", - "01715_background_checker_blather_zookeeper", - "01714_alter_drop_version", - "01114_materialize_clear_index_compact_parts", - "00814_replicated_minimalistic_part_header_zookeeper", - "01188_attach_table_from_pat", - "01415_sticking_mutations", - "01130_in_memory_parts", - "01110_dictionary_layout_without_arguments", - "01018_ddl_dictionaries_create", - "01018_ddl_dictionaries_select", - "01414_freeze_does_not_prevent_alters", - "01018_ddl_dictionaries_bad_queries", - "01686_rocksdb", - "01550_mutation_subquery", - "01070_mutations_with_dependencies", - "01070_materialize_ttl", - "01055_compact_parts", - "01017_mutations_with_nondeterministic_functions_zookeeper", - "00926_adaptive_index_granularity_pk", - "00910_zookeeper_test_alter_compression_codecs", - "00908_bloom_filter_index", - "00616_final_single_part", - "00446_clear_column_in_partition_zookeeper", - "01533_multiple_nested", - "01213_alter_rename_column_zookeeper", - "01575_disable_detach_table_of_dictionary", - "01457_create_as_table_function_structure", - "01415_inconsistent_merge_tree_settings", - "01413_allow_non_metadata_alters", - "01378_alter_rename_with_ttl_zookeeper", - "01349_mutation_datetime_key", - "01325_freeze_mutation_stuck", - "01272_suspicious_codecs", "01181_db_atomic_drop_on_cluster", - "00957_delta_diff_bug", - "00910_zookeeper_custom_compression_codecs_replicated", - "00899_long_attach_memory_limit", - "00804_test_custom_compression_codes_log_storages", - "00804_test_alter_compression_codecs", - "00804_test_delta_codec_no_type_alter", - "00804_test_custom_compression_codecs", - "00753_alter_attach", - "00715_fetch_merged_or_mutated_part_zookeeper", - "00688_low_cardinality_serialization", - "01575_disable_detach_table_of_dictionary", - "00738_lock_for_inner_table", - "01666_blns", - "01652_ignore_and_low_cardinality", - "01651_map_functions", - "01650_fetch_patition_with_macro_in_zk_path", - "01648_mutations_and_escaping", - "01640_marks_corruption_regression", - "01622_byte_size", - "01611_string_to_low_cardinality_key_alter", - "01602_show_create_view", - "01600_log_queries_with_extensive_info", - "01560_ttl_remove_empty_parts", - "01554_bloom_filter_index_big_integer_uuid", - "01550_type_map_formats_input", - "01550_type_map_formats", - "01550_create_map_type", - "01532_primary_key_without_order_by_zookeeper", - "01511_alter_version_versioned_collapsing_merge_tree_zookeeper", - "01509_parallel_quorum_insert_no_replicas", - "01504_compression_multiple_streams", - "01494_storage_join_persistency", - "01493_storage_set_persistency", - "01493_alter_remove_properties_zookeeper", - "01475_read_subcolumns_storages", - "01475_read_subcolumns", - "01451_replicated_detach_drop_part", - "01451_detach_drop_part", - "01440_big_int_exotic_casts", - "01430_modify_sample_by_zookeeper", - "01417_freeze_partition_verbose_zookeeper", - "01417_freeze_partition_verbose", - "01396_inactive_replica_cleanup_nodes_zookeeper", - "01375_compact_parts_codecs", - "01357_version_collapsing_attach_detach_zookeeper", - "01355_alter_column_with_order", - "01291_geo_types", - "01270_optimize_skip_unused_shards_low_cardinality", - "01182_materialized_view_different_structure", - "01150_ddl_guard_rwr", - "01148_zookeeper_path_macros_unfolding", - "01135_default_and_alter_zookeeper", - "01130_in_memory_parts_partitons", - "01127_month_partitioning_consistency_select", - "01114_database_atomic", - "01083_expressions_in_engine_arguments", - "01073_attach_if_not_exists", - "01072_optimize_skip_unused_shards_const_expr_eval", - "01071_prohibition_secondary_index_with_old_format_merge_tree", - "01062_alter_on_mutataion_zookeeper", - "01060_shutdown_table_after_detach", - "01056_create_table_as", - "01035_avg", - "01021_only_tuple_columns", - "01019_alter_materialized_view_query", - "01019_alter_materialized_view_consistent", - "01019_alter_materialized_view_atomic", - "01015_attach_part", - "00989_parallel_parts_loading", + "01175_distributed_ddl_output_mode", + "01415_sticking_mutations", "00980_zookeeper_merge_tree_alter_settings", - "00980_merge_alter_settings", + "01148_zookeeper_path_macros_unfolding", + "01019_alter_materialized_view_atomic", + "01019_alter_materialized_view_consistent", + "01018_ddl_dictionaries_bad_queries", "00955_test_final_mark", - "00933_reserved_word", - "00926_zookeeper_adaptive_index_granularity_replicated_merge_tree", - "00926_adaptive_index_granularity_replacing_merge_tree", - "00926_adaptive_index_granularity_merge_tree", - "00925_zookeeper_empty_replicated_merge_tree_optimize_final", - "00800_low_cardinality_distinct_numeric", - "00754_alter_modify_order_by_replicated_zookeeper", - "00751_low_cardinality_nullable_group_by", - "00751_default_databasename_for_view", - "00719_parallel_ddl_table", - "00718_low_cardinaliry_alter", - "00717_low_cardinaliry_distributed_group_by", - "00688_low_cardinality_syntax", - "00688_low_cardinality_nullable_cast", - "00688_low_cardinality_in", - "00652_replicated_mutations_zookeeper", - "00634_rename_view", + "00180_attach_materialized_view", + /// Unsupported type of ALTER query + "01650_fetch_patition_with_macro_in_zk_path", + "01451_detach_drop_part", + "01451_replicated_detach_drop_part", + "01417_freeze_partition_verbose", + "01417_freeze_partition_verbose_zookeeper", + "01130_in_memory_parts_partitons", + "01060_shutdown_table_after_detach", + "01021_only_tuple_columns", + "01015_attach_part", + "00626_replace_partition_from_table_zookeeper", "00626_replace_partition_from_table", - "00625_arrays_in_nested", + /// Old syntax is not allowed + "01062_alter_on_mutataion_zookeeper", + "00925_zookeeper_empty_replicated_merge_tree_optimize_final", + "00754_alter_modify_order_by_replicated_zookeeper", + "00652_replicated_mutations_zookeeper", "00623_replicated_truncate_table_zookeeper", - "00619_union_highlite", - "00599_create_view_with_subquery", - "00571_non_exist_database_when_create_materializ_view", - "00553_buff_exists_materlized_column", "00516_deduplication_after_drop_partition_zookeeper", - "00508_materialized_view_to", "00446_clear_column_in_partition_concurrent_zookeeper", - "00423_storage_log_single_thread", - "00311_array_primary_key", "00236_replicated_drop_on_non_leader_zookeeper", "00226_zookeeper_deduplication_and_unexpected_parts", "00215_primary_key_order_zookeeper", - "00180_attach_materialized_view", "00121_drop_column_zookeeper", - "00116_storage_set", "00083_create_merge_tree_zookeeper", "00062_replicated_merge_tree_alter_zookeeper", - "01720_constraints_complex_types", - "01747_alter_partition_key_enum_zookeeper" + /// Does not support renaming of multiple tables in single query + "00634_rename_view" ], "polymorphic-parts": [ "01508_partition_pruning_long", /// bug, shoud be fixed From 00b939e5a34c2a89132b196bfbab3505872db411 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 9 Mar 2021 13:24:08 +0300 Subject: [PATCH 030/464] fix --- src/Databases/DatabaseOnDisk.cpp | 6 +++--- src/Databases/DatabaseOnDisk.h | 2 +- src/Databases/DatabaseReplicated.cpp | 6 +++--- src/Databases/DatabaseReplicated.h | 2 +- src/Interpreters/DDLTask.cpp | 6 +++--- ...nce => 01175_distributed_ddl_output_mode_long.reference} | 0 ...ut_mode.sh => 01175_distributed_ddl_output_mode_long.sh} | 0 tests/queries/skip_list.json | 4 ++++ 8 files changed, 15 insertions(+), 11 deletions(-) rename tests/queries/0_stateless/{01175_distributed_ddl_output_mode.reference => 01175_distributed_ddl_output_mode_long.reference} (100%) rename tests/queries/0_stateless/{01175_distributed_ddl_output_mode.sh => 01175_distributed_ddl_output_mode_long.sh} (100%) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 50a7ce4df04..79373128713 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -231,7 +231,7 @@ void DatabaseOnDisk::createTable( if (create.attach_short_syntax) { /// Metadata already exists, table was detached - removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); + removeDetachedPermanentlyFlag(context, table_name, table_metadata_path, true); attachTable(table_name, table, getTableDataPath(create)); return; } @@ -270,12 +270,12 @@ void DatabaseOnDisk::createTable( commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path, context); - removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); + removeDetachedPermanentlyFlag(context, table_name, table_metadata_path, false); } /// If the table was detached permanently we will have a flag file with /// .sql.detached extension, is not needed anymore since we attached the table back -void DatabaseOnDisk::removeDetachedPermanentlyFlag(const Context &, const String & table_name, const String & table_metadata_path) const +void DatabaseOnDisk::removeDetachedPermanentlyFlag(const Context &, const String & table_name, const String & table_metadata_path, bool) const { try { diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index 8284cd92fdb..b96a24f3345 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -94,7 +94,7 @@ protected: virtual void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, const String & table_metadata_tmp_path, const String & table_metadata_path, const Context & query_context); - virtual void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const; + virtual void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path, bool attach) const; const String metadata_path; const String data_path; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index f075361ef7a..3125d95efa1 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -720,17 +720,17 @@ void DatabaseReplicated::detachTablePermanently(const Context & context, const S DatabaseAtomic::detachTablePermanently(context, table_name); } -void DatabaseReplicated::removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const +void DatabaseReplicated::removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path, bool attach) const { auto txn = context.getZooKeeperMetadataTransaction(); assert(!ddl_worker->isCurrentlyActive() || txn); - if (txn && txn->isInitialQuery() && !txn->isExecuted()) + if (txn && txn->isInitialQuery() && attach) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); String statement = readMetadataFile(table_name); txn->addOp(zkutil::makeCreateRequest(metadata_zk_path, statement, zkutil::CreateMode::Persistent)); } - DatabaseAtomic::removeDetachedPermanentlyFlag(context, table_name, table_metadata_path); + DatabaseAtomic::removeDetachedPermanentlyFlag(context, table_name, table_metadata_path, attach); } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 5290ea39ff6..fb46d997550 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -45,7 +45,7 @@ public: const ASTPtr & query) override; void removeDictionary(const Context & context, const String & dictionary_name) override; void detachTablePermanently(const Context & context, const String & table_name) override; - void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path) const override; + void removeDetachedPermanentlyFlag(const Context & context, const String & table_name, const String & table_metadata_path, bool attach) const override; /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 5d3f538f420..20d9a692a92 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -46,10 +46,10 @@ bool HostID::isLocalAddress(UInt16 clickhouse_port) const void DDLLogEntry::assertVersion() const { - constexpr UInt64 MAX_VERSION = 2; - if (version == 0 || MAX_VERSION < version) + constexpr UInt64 max_version = 2; + if (version == 0 || max_version < version) throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}." - "Maximum supported version is {}", version, MAX_VERSION); + "Maximum supported version is {}", version, max_version); } void DDLLogEntry::setSettingsIfRequired(const Context & context) diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference similarity index 100% rename from tests/queries/0_stateless/01175_distributed_ddl_output_mode.reference rename to tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh similarity index 100% rename from tests/queries/0_stateless/01175_distributed_ddl_output_mode.sh rename to tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 81e26675a1d..1c3e5b89e61 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -119,6 +119,8 @@ "01018_ddl_dictionaries_bad_queries", "00955_test_final_mark", "00180_attach_materialized_view", + "01294_system_distributed_on_cluster", + "01294_system_distributed_on_cluster", /// Unsupported type of ALTER query "01650_fetch_patition_with_macro_in_zk_path", "01451_detach_drop_part", @@ -131,6 +133,7 @@ "01015_attach_part", "00626_replace_partition_from_table_zookeeper", "00626_replace_partition_from_table", + "00152_insert_different_granularity", /// Old syntax is not allowed "01062_alter_on_mutataion_zookeeper", "00925_zookeeper_empty_replicated_merge_tree_optimize_final", @@ -627,6 +630,7 @@ "01601_detach_permanently", "01602_show_create_view", "01603_rename_overwrite_bug", + "01666_blns", "01646_system_restart_replicas_smoke", // system restart replicas is a global query "01656_test_query_log_factories_info", "01669_columns_declaration_serde", From b936619fa963a128633b960b9b1b7134f2ed9307 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 9 Mar 2021 20:09:35 +0800 Subject: [PATCH 031/464] Rename --- .../{connectionID.cpp => connectionId.cpp} | 16 ++++++++-------- .../{partitionID.cpp => partitionId.cpp} | 12 ++++++------ src/Functions/registerFunctionsMiscellaneous.cpp | 8 ++++---- .../0_stateless/01748_partition_id_pruning.sql | 6 +++--- 4 files changed, 21 insertions(+), 21 deletions(-) rename src/Functions/{connectionID.cpp => connectionId.cpp} (64%) rename src/Functions/{partitionID.cpp => partitionId.cpp} (86%) diff --git a/src/Functions/connectionID.cpp b/src/Functions/connectionId.cpp similarity index 64% rename from src/Functions/connectionID.cpp rename to src/Functions/connectionId.cpp index 8e9c81aed6c..a2587afdb1b 100644 --- a/src/Functions/connectionID.cpp +++ b/src/Functions/connectionId.cpp @@ -7,15 +7,15 @@ namespace DB { -/// Get the connection ID. It's used for MySQL handler only. -class FunctionConnectionID : public IFunction +/// Get the connection Id. It's used for MySQL handler only. +class FunctionConnectionId : public IFunction { public: - static constexpr auto name = "connectionID"; + static constexpr auto name = "connectionId"; - explicit FunctionConnectionID(const Context & context_) : context(context_) {} + explicit FunctionConnectionId(const Context & context_) : context(context_) {} - static FunctionPtr create(const Context & context) { return std::make_shared(context); } + static FunctionPtr create(const Context & context) { return std::make_shared(context); } String getName() const override { return name; } @@ -32,10 +32,10 @@ private: const Context & context; }; -void registerFunctionConnectionID(FunctionFactory & factory) +void registerFunctionConnectionId(FunctionFactory & factory) { - factory.registerFunction(); - factory.registerAlias("connection_id", "connectionID"); + factory.registerFunction(); + factory.registerAlias("connection_id", "connectionId"); } } diff --git a/src/Functions/partitionID.cpp b/src/Functions/partitionId.cpp similarity index 86% rename from src/Functions/partitionID.cpp rename to src/Functions/partitionId.cpp index 9f5a3d09b86..b6d9d1bf4e6 100644 --- a/src/Functions/partitionID.cpp +++ b/src/Functions/partitionId.cpp @@ -15,14 +15,14 @@ namespace ErrorCodes } -/** partitionID(x, y, ...) is a function that computes partition ids of arguments. +/** partitionId(x, y, ...) is a function that computes partition ids of arguments. */ -class FunctionPartitionID : public IFunction +class FunctionPartitionId : public IFunction { public: - static constexpr auto name = "partitionID"; + static constexpr auto name = "partitionId"; - static FunctionPtr create(const Context &) { return std::make_shared(); } + static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { return name; } @@ -67,9 +67,9 @@ public: } }; -void registerFunctionPartitionID(FunctionFactory & factory) +void registerFunctionPartitionId(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 3557b0fb865..f2c35420ab6 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -69,8 +69,8 @@ void registerFunctionErrorCodeToName(FunctionFactory &); void registerFunctionTcpPort(FunctionFactory &); void registerFunctionByteSize(FunctionFactory &); void registerFunctionFile(FunctionFactory & factory); -void registerFunctionConnectionID(FunctionFactory & factory); -void registerFunctionPartitionID(FunctionFactory & factory); +void registerFunctionConnectionId(FunctionFactory & factory); +void registerFunctionPartitionId(FunctionFactory & factory); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -140,8 +140,8 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionTcpPort(factory); registerFunctionByteSize(factory); registerFunctionFile(factory); - registerFunctionConnectionID(factory); - registerFunctionPartitionID(factory); + registerFunctionConnectionId(factory); + registerFunctionPartitionId(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index cca366c5168..0a3f7d2713c 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -6,14 +6,14 @@ insert into x values (1, 1), (1, 2), (1, 3), (2, 4), (2, 5), (2, 6); set max_rows_to_read = 3; -select * from x where _partition_id = partitionID(1); +select * from x where _partition_id = partitionId(1); set max_rows_to_read = 4; -- one row for subquery -select * from x where _partition_id in (select partitionID(number + 1) from numbers(1)); +select * from x where _partition_id in (select partitionId(number + 1) from numbers(1)); -- trivial count optimization test set max_rows_to_read = 1; -- one row for subquery -select count() from x where _partition_id in (select partitionID(number + 1) from numbers(1)); +select count() from x where _partition_id in (select partitionId(number + 1) from numbers(1)); drop table x; From 1e61f64f9545b373347a13088d2c39edb8cea1b7 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 9 Mar 2021 17:04:20 +0300 Subject: [PATCH 032/464] move data types to serializations --- .../SerializationAggregateFunction.cpp} | 0 .../SerializationAggregateFunction.h} | 0 .../{DataTypeArray.cpp => Serializations/SerializationArray.cpp} | 0 .../{DataTypeArray.h => Serializations/SerializationArray.h} | 0 .../{DataTypeDate.cpp => Serializations/SerializationDate.cpp} | 0 .../{DataTypeDate.h => Serializations/SerializationDate.h} | 0 .../SerializationDateTime.cpp} | 0 .../SerializationDateTime.h} | 0 .../SerializationDateTime64.cpp} | 0 .../SerializationDateTime64.h} | 0 .../SerializationDecimalBase.cpp} | 0 .../SerializationDecimalBase.h} | 0 .../{DataTypeEnum.cpp => Serializations/SerializationEnum.cpp} | 0 .../{DataTypeEnum.h => Serializations/SerializationEnum.h} | 0 .../SerializationFixedString.cpp} | 0 .../SerializationFixedString.h} | 0 .../SerializationLowCardinality.cpp} | 0 .../SerializationLowCardinality.h} | 0 .../{DataTypeMap.cpp => Serializations/SerializationMap.cpp} | 0 .../{DataTypeMap.h => Serializations/SerializationMap.h} | 0 .../SerializationNothing.cpp} | 0 .../{DataTypeNothing.h => Serializations/SerializationNothing.h} | 0 .../SerializationNullable.cpp} | 0 .../SerializationNullable.h} | 0 .../SerializationString.cpp} | 0 .../{DataTypeString.h => Serializations/SerializationString.h} | 0 .../{DataTypeTuple.cpp => Serializations/SerializationTuple.cpp} | 0 .../{DataTypeTuple.h => Serializations/SerializationTuple.h} | 0 .../{DataTypeUUID.cpp => Serializations/SerializationUUID.cpp} | 0 .../{DataTypeUUID.h => Serializations/SerializationUUID.h} | 0 30 files changed, 0 insertions(+), 0 deletions(-) rename src/DataTypes/{DataTypeAggregateFunction.cpp => Serializations/SerializationAggregateFunction.cpp} (100%) rename src/DataTypes/{DataTypeAggregateFunction.h => Serializations/SerializationAggregateFunction.h} (100%) rename src/DataTypes/{DataTypeArray.cpp => Serializations/SerializationArray.cpp} (100%) rename src/DataTypes/{DataTypeArray.h => Serializations/SerializationArray.h} (100%) rename src/DataTypes/{DataTypeDate.cpp => Serializations/SerializationDate.cpp} (100%) rename src/DataTypes/{DataTypeDate.h => Serializations/SerializationDate.h} (100%) rename src/DataTypes/{DataTypeDateTime.cpp => Serializations/SerializationDateTime.cpp} (100%) rename src/DataTypes/{DataTypeDateTime.h => Serializations/SerializationDateTime.h} (100%) rename src/DataTypes/{DataTypeDateTime64.cpp => Serializations/SerializationDateTime64.cpp} (100%) rename src/DataTypes/{DataTypeDateTime64.h => Serializations/SerializationDateTime64.h} (100%) rename src/DataTypes/{DataTypeDecimalBase.cpp => Serializations/SerializationDecimalBase.cpp} (100%) rename src/DataTypes/{DataTypeDecimalBase.h => Serializations/SerializationDecimalBase.h} (100%) rename src/DataTypes/{DataTypeEnum.cpp => Serializations/SerializationEnum.cpp} (100%) rename src/DataTypes/{DataTypeEnum.h => Serializations/SerializationEnum.h} (100%) rename src/DataTypes/{DataTypeFixedString.cpp => Serializations/SerializationFixedString.cpp} (100%) rename src/DataTypes/{DataTypeFixedString.h => Serializations/SerializationFixedString.h} (100%) rename src/DataTypes/{DataTypeLowCardinality.cpp => Serializations/SerializationLowCardinality.cpp} (100%) rename src/DataTypes/{DataTypeLowCardinality.h => Serializations/SerializationLowCardinality.h} (100%) rename src/DataTypes/{DataTypeMap.cpp => Serializations/SerializationMap.cpp} (100%) rename src/DataTypes/{DataTypeMap.h => Serializations/SerializationMap.h} (100%) rename src/DataTypes/{DataTypeNothing.cpp => Serializations/SerializationNothing.cpp} (100%) rename src/DataTypes/{DataTypeNothing.h => Serializations/SerializationNothing.h} (100%) rename src/DataTypes/{DataTypeNullable.cpp => Serializations/SerializationNullable.cpp} (100%) rename src/DataTypes/{DataTypeNullable.h => Serializations/SerializationNullable.h} (100%) rename src/DataTypes/{DataTypeString.cpp => Serializations/SerializationString.cpp} (100%) rename src/DataTypes/{DataTypeString.h => Serializations/SerializationString.h} (100%) rename src/DataTypes/{DataTypeTuple.cpp => Serializations/SerializationTuple.cpp} (100%) rename src/DataTypes/{DataTypeTuple.h => Serializations/SerializationTuple.h} (100%) rename src/DataTypes/{DataTypeUUID.cpp => Serializations/SerializationUUID.cpp} (100%) rename src/DataTypes/{DataTypeUUID.h => Serializations/SerializationUUID.h} (100%) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp similarity index 100% rename from src/DataTypes/DataTypeAggregateFunction.cpp rename to src/DataTypes/Serializations/SerializationAggregateFunction.cpp diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/Serializations/SerializationAggregateFunction.h similarity index 100% rename from src/DataTypes/DataTypeAggregateFunction.h rename to src/DataTypes/Serializations/SerializationAggregateFunction.h diff --git a/src/DataTypes/DataTypeArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp similarity index 100% rename from src/DataTypes/DataTypeArray.cpp rename to src/DataTypes/Serializations/SerializationArray.cpp diff --git a/src/DataTypes/DataTypeArray.h b/src/DataTypes/Serializations/SerializationArray.h similarity index 100% rename from src/DataTypes/DataTypeArray.h rename to src/DataTypes/Serializations/SerializationArray.h diff --git a/src/DataTypes/DataTypeDate.cpp b/src/DataTypes/Serializations/SerializationDate.cpp similarity index 100% rename from src/DataTypes/DataTypeDate.cpp rename to src/DataTypes/Serializations/SerializationDate.cpp diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/Serializations/SerializationDate.h similarity index 100% rename from src/DataTypes/DataTypeDate.h rename to src/DataTypes/Serializations/SerializationDate.h diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/Serializations/SerializationDateTime.cpp similarity index 100% rename from src/DataTypes/DataTypeDateTime.cpp rename to src/DataTypes/Serializations/SerializationDateTime.cpp diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/Serializations/SerializationDateTime.h similarity index 100% rename from src/DataTypes/DataTypeDateTime.h rename to src/DataTypes/Serializations/SerializationDateTime.h diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/Serializations/SerializationDateTime64.cpp similarity index 100% rename from src/DataTypes/DataTypeDateTime64.cpp rename to src/DataTypes/Serializations/SerializationDateTime64.cpp diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/Serializations/SerializationDateTime64.h similarity index 100% rename from src/DataTypes/DataTypeDateTime64.h rename to src/DataTypes/Serializations/SerializationDateTime64.h diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/Serializations/SerializationDecimalBase.cpp similarity index 100% rename from src/DataTypes/DataTypeDecimalBase.cpp rename to src/DataTypes/Serializations/SerializationDecimalBase.cpp diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/Serializations/SerializationDecimalBase.h similarity index 100% rename from src/DataTypes/DataTypeDecimalBase.h rename to src/DataTypes/Serializations/SerializationDecimalBase.h diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/Serializations/SerializationEnum.cpp similarity index 100% rename from src/DataTypes/DataTypeEnum.cpp rename to src/DataTypes/Serializations/SerializationEnum.cpp diff --git a/src/DataTypes/DataTypeEnum.h b/src/DataTypes/Serializations/SerializationEnum.h similarity index 100% rename from src/DataTypes/DataTypeEnum.h rename to src/DataTypes/Serializations/SerializationEnum.h diff --git a/src/DataTypes/DataTypeFixedString.cpp b/src/DataTypes/Serializations/SerializationFixedString.cpp similarity index 100% rename from src/DataTypes/DataTypeFixedString.cpp rename to src/DataTypes/Serializations/SerializationFixedString.cpp diff --git a/src/DataTypes/DataTypeFixedString.h b/src/DataTypes/Serializations/SerializationFixedString.h similarity index 100% rename from src/DataTypes/DataTypeFixedString.h rename to src/DataTypes/Serializations/SerializationFixedString.h diff --git a/src/DataTypes/DataTypeLowCardinality.cpp b/src/DataTypes/Serializations/SerializationLowCardinality.cpp similarity index 100% rename from src/DataTypes/DataTypeLowCardinality.cpp rename to src/DataTypes/Serializations/SerializationLowCardinality.cpp diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/Serializations/SerializationLowCardinality.h similarity index 100% rename from src/DataTypes/DataTypeLowCardinality.h rename to src/DataTypes/Serializations/SerializationLowCardinality.h diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp similarity index 100% rename from src/DataTypes/DataTypeMap.cpp rename to src/DataTypes/Serializations/SerializationMap.cpp diff --git a/src/DataTypes/DataTypeMap.h b/src/DataTypes/Serializations/SerializationMap.h similarity index 100% rename from src/DataTypes/DataTypeMap.h rename to src/DataTypes/Serializations/SerializationMap.h diff --git a/src/DataTypes/DataTypeNothing.cpp b/src/DataTypes/Serializations/SerializationNothing.cpp similarity index 100% rename from src/DataTypes/DataTypeNothing.cpp rename to src/DataTypes/Serializations/SerializationNothing.cpp diff --git a/src/DataTypes/DataTypeNothing.h b/src/DataTypes/Serializations/SerializationNothing.h similarity index 100% rename from src/DataTypes/DataTypeNothing.h rename to src/DataTypes/Serializations/SerializationNothing.h diff --git a/src/DataTypes/DataTypeNullable.cpp b/src/DataTypes/Serializations/SerializationNullable.cpp similarity index 100% rename from src/DataTypes/DataTypeNullable.cpp rename to src/DataTypes/Serializations/SerializationNullable.cpp diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/Serializations/SerializationNullable.h similarity index 100% rename from src/DataTypes/DataTypeNullable.h rename to src/DataTypes/Serializations/SerializationNullable.h diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/Serializations/SerializationString.cpp similarity index 100% rename from src/DataTypes/DataTypeString.cpp rename to src/DataTypes/Serializations/SerializationString.cpp diff --git a/src/DataTypes/DataTypeString.h b/src/DataTypes/Serializations/SerializationString.h similarity index 100% rename from src/DataTypes/DataTypeString.h rename to src/DataTypes/Serializations/SerializationString.h diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp similarity index 100% rename from src/DataTypes/DataTypeTuple.cpp rename to src/DataTypes/Serializations/SerializationTuple.cpp diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/Serializations/SerializationTuple.h similarity index 100% rename from src/DataTypes/DataTypeTuple.h rename to src/DataTypes/Serializations/SerializationTuple.h diff --git a/src/DataTypes/DataTypeUUID.cpp b/src/DataTypes/Serializations/SerializationUUID.cpp similarity index 100% rename from src/DataTypes/DataTypeUUID.cpp rename to src/DataTypes/Serializations/SerializationUUID.cpp diff --git a/src/DataTypes/DataTypeUUID.h b/src/DataTypes/Serializations/SerializationUUID.h similarity index 100% rename from src/DataTypes/DataTypeUUID.h rename to src/DataTypes/Serializations/SerializationUUID.h From be540e442d684efed7cb6f3bc64ebb24f0e6cb60 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 9 Mar 2021 17:10:28 +0300 Subject: [PATCH 033/464] return back data types --- src/DataTypes/DataTypeAggregateFunction.cpp | 173 +++++++ src/DataTypes/DataTypeAggregateFunction.h | 57 +++ src/DataTypes/DataTypeArray.cpp | 147 ++++++ src/DataTypes/DataTypeArray.h | 76 ++++ src/DataTypes/DataTypeCustom.h | 94 +--- src/DataTypes/DataTypeCustomGeo.cpp | 96 +--- src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp | 99 +--- src/DataTypes/DataTypeCustom_fwd.h | 26 +- src/DataTypes/DataTypeDate.cpp | 30 ++ src/DataTypes/DataTypeDate.h | 26 ++ src/DataTypes/DataTypeDateTime.cpp | 56 +++ src/DataTypes/DataTypeDateTime.h | 70 +++ src/DataTypes/DataTypeDateTime64.cpp | 71 +++ src/DataTypes/DataTypeDateTime64.h | 104 +++++ src/DataTypes/DataTypeDecimalBase.cpp | 52 +++ src/DataTypes/DataTypeDecimalBase.h | 206 +++++++++ src/DataTypes/DataTypeEnum.cpp | 267 +++++++++++ src/DataTypes/DataTypeEnum.h | 88 ++++ src/DataTypes/DataTypeFactory.h | 3 +- src/DataTypes/DataTypeFixedString.cpp | 79 ++++ src/DataTypes/DataTypeFixedString.h | 69 +++ src/DataTypes/DataTypeLowCardinality.cpp | 167 +++++++ src/DataTypes/DataTypeLowCardinality.h | 89 ++++ src/DataTypes/DataTypeMap.cpp | 125 ++++++ src/DataTypes/DataTypeMap.h | 58 +++ src/DataTypes/DataTypeNested.cpp | 2 +- src/DataTypes/DataTypeNested.h | 3 +- src/DataTypes/DataTypeNothing.cpp | 34 ++ src/DataTypes/DataTypeNothing.h | 34 ++ src/DataTypes/DataTypeNullable.cpp | 129 ++++++ src/DataTypes/DataTypeNullable.h | 60 +++ src/DataTypes/DataTypeNumberBase.cpp | 184 -------- src/DataTypes/DataTypeNumberBase.h | 21 +- src/DataTypes/DataTypeString.cpp | 112 +++++ src/DataTypes/DataTypeString.h | 43 ++ src/DataTypes/DataTypeTuple.cpp | 391 ++++++++++++++++ src/DataTypes/DataTypeTuple.h | 81 ++++ src/DataTypes/DataTypeUUID.cpp | 24 + src/DataTypes/DataTypeUUID.h | 28 ++ src/DataTypes/DataTypesDecimal.cpp | 56 +-- src/DataTypes/DataTypesDecimal.h | 10 +- src/DataTypes/DataTypesNumber.h | 6 + src/DataTypes/EnumValues.cpp | 77 ++++ src/DataTypes/EnumValues.h | 71 +++ src/DataTypes/IDataType.cpp | 424 +++++------------- src/DataTypes/IDataType.h | 297 ++---------- src/DataTypes/IDataTypeDummy.h | 15 +- .../Serializations/ISerialization.cpp | 197 ++++++++ src/DataTypes/Serializations/ISerialization.h | 266 +++++++++++ .../SerializationCustomSimpleText.cpp | 97 ++++ .../SerializationCustomSimpleText.h | 57 +++ .../Serializations/SerializationDecimal.cpp | 74 +++ .../Serializations/SerializationDecimal.h | 27 ++ .../Serializations/SerializationIP.cpp | 93 ++++ .../Serializations/SerializationIP.h | 26 ++ .../Serializations/SerializationInfo.cpp | 122 +++++ .../Serializations/SerializationInfo.h | 32 ++ .../Serializations/SerializationNumber.cpp | 215 +++++++++ .../Serializations/SerializationNumber.h | 33 ++ .../Serializations/SerializationSparse.cpp | 155 +++++++ .../Serializations/SerializationSparse.h | 42 ++ .../SerializationTupleElement.cpp | 73 +++ .../SerializationTupleElement.h | 57 +++ .../Serializations/SerializationWrapper.cpp | 140 ++++++ .../Serializations/SerializationWrapper.h | 72 +++ .../Serializations/SimpleTextSerialization.h | 63 +++ .../gtest_DataType_deserializeAsText.cpp | 2 +- src/DataTypes/ya.make | 24 +- 68 files changed, 5151 insertions(+), 1146 deletions(-) create mode 100644 src/DataTypes/DataTypeAggregateFunction.cpp create mode 100644 src/DataTypes/DataTypeAggregateFunction.h create mode 100644 src/DataTypes/DataTypeArray.cpp create mode 100644 src/DataTypes/DataTypeArray.h create mode 100644 src/DataTypes/DataTypeDate.cpp create mode 100644 src/DataTypes/DataTypeDate.h create mode 100644 src/DataTypes/DataTypeDateTime.cpp create mode 100644 src/DataTypes/DataTypeDateTime.h create mode 100644 src/DataTypes/DataTypeDateTime64.cpp create mode 100644 src/DataTypes/DataTypeDateTime64.h create mode 100644 src/DataTypes/DataTypeDecimalBase.cpp create mode 100644 src/DataTypes/DataTypeDecimalBase.h create mode 100644 src/DataTypes/DataTypeEnum.cpp create mode 100644 src/DataTypes/DataTypeEnum.h create mode 100644 src/DataTypes/DataTypeFixedString.cpp create mode 100644 src/DataTypes/DataTypeFixedString.h create mode 100644 src/DataTypes/DataTypeLowCardinality.cpp create mode 100644 src/DataTypes/DataTypeLowCardinality.h create mode 100644 src/DataTypes/DataTypeMap.cpp create mode 100644 src/DataTypes/DataTypeMap.h create mode 100644 src/DataTypes/DataTypeNothing.cpp create mode 100644 src/DataTypes/DataTypeNothing.h create mode 100644 src/DataTypes/DataTypeNullable.cpp create mode 100644 src/DataTypes/DataTypeNullable.h create mode 100644 src/DataTypes/DataTypeString.cpp create mode 100644 src/DataTypes/DataTypeString.h create mode 100644 src/DataTypes/DataTypeTuple.cpp create mode 100644 src/DataTypes/DataTypeTuple.h create mode 100644 src/DataTypes/DataTypeUUID.cpp create mode 100644 src/DataTypes/DataTypeUUID.h create mode 100644 src/DataTypes/EnumValues.cpp create mode 100644 src/DataTypes/EnumValues.h create mode 100644 src/DataTypes/Serializations/ISerialization.cpp create mode 100644 src/DataTypes/Serializations/ISerialization.h create mode 100644 src/DataTypes/Serializations/SerializationCustomSimpleText.cpp create mode 100644 src/DataTypes/Serializations/SerializationCustomSimpleText.h create mode 100644 src/DataTypes/Serializations/SerializationDecimal.cpp create mode 100644 src/DataTypes/Serializations/SerializationDecimal.h create mode 100644 src/DataTypes/Serializations/SerializationIP.cpp create mode 100644 src/DataTypes/Serializations/SerializationIP.h create mode 100644 src/DataTypes/Serializations/SerializationInfo.cpp create mode 100644 src/DataTypes/Serializations/SerializationInfo.h create mode 100644 src/DataTypes/Serializations/SerializationNumber.cpp create mode 100644 src/DataTypes/Serializations/SerializationNumber.h create mode 100644 src/DataTypes/Serializations/SerializationSparse.cpp create mode 100644 src/DataTypes/Serializations/SerializationSparse.h create mode 100644 src/DataTypes/Serializations/SerializationTupleElement.cpp create mode 100644 src/DataTypes/Serializations/SerializationTupleElement.h create mode 100644 src/DataTypes/Serializations/SerializationWrapper.cpp create mode 100644 src/DataTypes/Serializations/SerializationWrapper.h create mode 100644 src/DataTypes/Serializations/SimpleTextSerialization.h diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp new file mode 100644 index 00000000000..7f7b01e031b --- /dev/null +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; + extern const int BAD_ARGUMENTS; + extern const int PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; +} + + +std::string DataTypeAggregateFunction::doGetName() const +{ + WriteBufferFromOwnString stream; + stream << "AggregateFunction(" << function->getName(); + + if (!parameters.empty()) + { + stream << '('; + for (size_t i = 0; i < parameters.size(); ++i) + { + if (i) + stream << ", "; + stream << applyVisitor(DB::FieldVisitorToString(), parameters[i]); + } + stream << ')'; + } + + for (const auto & argument_type : argument_types) + stream << ", " << argument_type->getName(); + + stream << ')'; + return stream.str(); +} + +MutableColumnPtr DataTypeAggregateFunction::createColumn() const +{ + return ColumnAggregateFunction::create(function); +} + + +/// Create empty state +Field DataTypeAggregateFunction::getDefault() const +{ + Field field = AggregateFunctionStateData(); + field.get().name = getName(); + + AlignedBuffer place_buffer(function->sizeOfData(), function->alignOfData()); + AggregateDataPtr place = place_buffer.data(); + + function->create(place); + + try + { + WriteBufferFromString buffer_from_field(field.get().data); + function->serialize(place, buffer_from_field); + } + catch (...) + { + function->destroy(place); + throw; + } + + function->destroy(place); + + return field; +} + + +bool DataTypeAggregateFunction::equals(const IDataType & rhs) const +{ + return typeid(rhs) == typeid(*this) && getName() == rhs.getName(); +} + +SerializationPtr DataTypeAggregateFunction::doGetDefaultSerialization() const +{ + return std::make_shared(function); +} + + +static DataTypePtr create(const ASTPtr & arguments) +{ + String function_name; + AggregateFunctionPtr function; + DataTypes argument_types; + Array params_row; + + if (!arguments || arguments->children.empty()) + throw Exception("Data type AggregateFunction requires parameters: " + "name of aggregate function and list of data types for arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (const auto * parametric = arguments->children[0]->as()) + { + if (parametric->parameters) + throw Exception("Unexpected level of parameters to aggregate function", ErrorCodes::SYNTAX_ERROR); + function_name = parametric->name; + + if (parametric->arguments) + { + const ASTs & parameters = parametric->arguments->children; + params_row.resize(parameters.size()); + + for (size_t i = 0; i < parameters.size(); ++i) + { + const auto * literal = parameters[i]->as(); + if (!literal) + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}' for function '{}'", + parameters[i]->formatForErrorMessage(), function_name); + + params_row[i] = literal->value; + } + } + } + else if (auto opt_name = tryGetIdentifierName(arguments->children[0])) + { + function_name = *opt_name; + } + else if (arguments->children[0]->as()) + { + throw Exception("Aggregate function name for data type AggregateFunction must be passed as identifier (without quotes) or function", + ErrorCodes::BAD_ARGUMENTS); + } + else + throw Exception("Unexpected AST element passed as aggregate function name for data type AggregateFunction. Must be identifier or function.", + ErrorCodes::BAD_ARGUMENTS); + + for (size_t i = 1; i < arguments->children.size(); ++i) + argument_types.push_back(DataTypeFactory::instance().get(arguments->children[i])); + + if (function_name.empty()) + throw Exception("Logical error: empty name of aggregate function passed", ErrorCodes::LOGICAL_ERROR); + + AggregateFunctionProperties properties; + function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row, properties); + return std::make_shared(function, argument_types, params_row); +} + +void registerDataTypeAggregateFunction(DataTypeFactory & factory) +{ + factory.registerDataType("AggregateFunction", create); +} + + +} diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h new file mode 100644 index 00000000000..c3fea2ba727 --- /dev/null +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include + + +namespace DB +{ + +/** Type - the state of the aggregate function. + * Type parameters is an aggregate function, the types of its arguments, and its parameters (for parametric aggregate functions). + */ +class DataTypeAggregateFunction final : public IDataType +{ +private: + AggregateFunctionPtr function; + DataTypes argument_types; + Array parameters; + +public: + static constexpr bool is_parametric = true; + + DataTypeAggregateFunction(const AggregateFunctionPtr & function_, const DataTypes & argument_types_, const Array & parameters_) + : function(function_), argument_types(argument_types_), parameters(parameters_) + { + } + + std::string getFunctionName() const { return function->getName(); } + AggregateFunctionPtr getFunction() const { return function; } + + std::string doGetName() const override; + const char * getFamilyName() const override { return "AggregateFunction"; } + TypeIndex getTypeId() const override { return TypeIndex::AggregateFunction; } + + bool canBeInsideNullable() const override { return false; } + + DataTypePtr getReturnType() const { return function->getReturnType(); } + DataTypePtr getReturnTypeToPredict() const { return function->getReturnTypeToPredict(); } + DataTypes getArgumentsDataTypes() const { return argument_types; } + + MutableColumnPtr createColumn() const override; + + Field getDefault() const override; + + bool equals(const IDataType & rhs) const override; + + bool isParametric() const override { return true; } + bool haveSubtypes() const override { return false; } + bool shouldAlignRightInPrettyFormats() const override { return false; } + + SerializationPtr doGetDefaultSerialization() const override; +}; + + +} + diff --git a/src/DataTypes/DataTypeArray.cpp b/src/DataTypes/DataTypeArray.cpp new file mode 100644 index 00000000000..bcf3a9c1f57 --- /dev/null +++ b/src/DataTypes/DataTypeArray.cpp @@ -0,0 +1,147 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +DataTypeArray::DataTypeArray(const DataTypePtr & nested_) + : nested{nested_} +{ +} + + +MutableColumnPtr DataTypeArray::createColumn() const +{ + return ColumnArray::create(nested->createColumn(), ColumnArray::ColumnOffsets::create()); +} + + +Field DataTypeArray::getDefault() const +{ + return Array(); +} + + +bool DataTypeArray::equals(const IDataType & rhs) const +{ + return typeid(rhs) == typeid(*this) && nested->equals(*static_cast(rhs).nested); +} + +DataTypePtr DataTypeArray::tryGetSubcolumnType(const String & subcolumn_name) const +{ + return tryGetSubcolumnTypeImpl(subcolumn_name, 0); +} + +DataTypePtr DataTypeArray::tryGetSubcolumnTypeImpl(const String & subcolumn_name, size_t level) const +{ + if (subcolumn_name == "size" + std::to_string(level)) + return std::make_shared(); + + DataTypePtr subcolumn; + if (const auto * nested_array = typeid_cast(nested.get())) + subcolumn = nested_array->tryGetSubcolumnTypeImpl(subcolumn_name, level + 1); + else + subcolumn = nested->tryGetSubcolumnType(subcolumn_name); + + if (subcolumn && subcolumn_name != MAIN_SUBCOLUMN_NAME) + subcolumn = std::make_shared(std::move(subcolumn)); + + return subcolumn; +} + +ColumnPtr DataTypeArray::getSubcolumn(const String & subcolumn_name, const IColumn & column) const +{ + return getSubcolumnImpl(subcolumn_name, column, 0); +} + +ColumnPtr DataTypeArray::getSubcolumnImpl(const String & subcolumn_name, const IColumn & column, size_t level) const +{ + const auto & column_array = assert_cast(column); + if (subcolumn_name == "size" + std::to_string(level)) + return arrayOffsetsToSizes(column_array.getOffsetsColumn()); + + ColumnPtr subcolumn; + if (const auto * nested_array = typeid_cast(nested.get())) + subcolumn = nested_array->getSubcolumnImpl(subcolumn_name, column_array.getData(), level + 1); + else + subcolumn = nested->getSubcolumn(subcolumn_name, column_array.getData()); + + return ColumnArray::create(subcolumn, column_array.getOffsetsPtr()); +} + +SerializationPtr DataTypeArray::getSubcolumnSerialization( + const String & subcolumn_name, const BaseSerializationGetter & base_serialization_getter) const +{ + return getSubcolumnSerializationImpl(subcolumn_name, base_serialization_getter, 0); +} + +SerializationPtr DataTypeArray::getSubcolumnSerializationImpl( + const String & subcolumn_name, const BaseSerializationGetter & base_serialization_getter, size_t level) const +{ + if (subcolumn_name == "size" + std::to_string(level)) + return std::make_shared(base_serialization_getter(DataTypeUInt64()), subcolumn_name, false); + + SerializationPtr subcolumn; + if (const auto * nested_array = typeid_cast(nested.get())) + subcolumn = nested_array->getSubcolumnSerializationImpl(subcolumn_name, base_serialization_getter, level + 1); + else + subcolumn = nested->getSubcolumnSerialization(subcolumn_name, base_serialization_getter); + + return std::make_shared(subcolumn); +} + +SerializationPtr DataTypeArray::doGetDefaultSerialization() const +{ + return std::make_shared(nested->getDefaultSerialization()); +} + +size_t DataTypeArray::getNumberOfDimensions() const +{ + const DataTypeArray * nested_array = typeid_cast(nested.get()); + if (!nested_array) + return 1; + return 1 + nested_array->getNumberOfDimensions(); /// Every modern C++ compiler optimizes tail recursion. +} + + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (!arguments || arguments->children.size() != 1) + throw Exception("Array data type family must have exactly one argument - type of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(DataTypeFactory::instance().get(arguments->children[0])); +} + + +void registerDataTypeArray(DataTypeFactory & factory) +{ + factory.registerDataType("Array", create); +} + +} diff --git a/src/DataTypes/DataTypeArray.h b/src/DataTypes/DataTypeArray.h new file mode 100644 index 00000000000..c720a15d798 --- /dev/null +++ b/src/DataTypes/DataTypeArray.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + + +namespace DB +{ + + +class DataTypeArray final : public IDataType +{ +private: + /// The type of array elements. + DataTypePtr nested; + +public: + static constexpr bool is_parametric = true; + + DataTypeArray(const DataTypePtr & nested_); + + TypeIndex getTypeId() const override { return TypeIndex::Array; } + + std::string doGetName() const override + { + return "Array(" + nested->getName() + ")"; + } + + const char * getFamilyName() const override + { + return "Array"; + } + + bool canBeInsideNullable() const override + { + return false; + } + + MutableColumnPtr createColumn() const override; + + Field getDefault() const override; + + bool equals(const IDataType & rhs) const override; + + bool isParametric() const override { return true; } + bool haveSubtypes() const override { return true; } + bool cannotBeStoredInTables() const override { return nested->cannotBeStoredInTables(); } + bool textCanContainOnlyValidUTF8() const override { return nested->textCanContainOnlyValidUTF8(); } + bool isComparable() const override { return nested->isComparable(); } + bool canBeComparedWithCollation() const override { return nested->canBeComparedWithCollation(); } + + bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override + { + return nested->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion(); + } + + DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const override; + ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const override; + SerializationPtr getSubcolumnSerialization( + const String & subcolumn_name, const BaseSerializationGetter & base_serialization_getter) const override; + + SerializationPtr doGetDefaultSerialization() const override; + + const DataTypePtr & getNestedType() const { return nested; } + + /// 1 for plain array, 2 for array of arrays and so on. + size_t getNumberOfDimensions() const; + +private: + ColumnPtr getSubcolumnImpl(const String & subcolumn_name, const IColumn & column, size_t level) const; + DataTypePtr tryGetSubcolumnTypeImpl(const String & subcolumn_name, size_t level) const; + SerializationPtr getSubcolumnSerializationImpl( + const String & subcolumn_name, const BaseSerializationGetter & base_serialization_getter, size_t level) const; +}; + +} diff --git a/src/DataTypes/DataTypeCustom.h b/src/DataTypes/DataTypeCustom.h index 0fa2e365990..55796e3cc7a 100644 --- a/src/DataTypes/DataTypeCustom.h +++ b/src/DataTypes/DataTypeCustom.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB { @@ -24,106 +24,20 @@ public: virtual String getName() const = 0; }; -class IDataTypeCustomTextSerialization -{ -public: - virtual ~IDataTypeCustomTextSerialization() {} - - /** Text serialization for displaying on a terminal or saving into a text file, and the like. - * Without escaping or quoting. - */ - virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; - - /** Text deserialization without quoting or escaping. - */ - virtual void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; - - /** Text serialization with escaping but without quoting. - */ - virtual void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; - virtual void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; - - /** Text serialization as a literal that may be inserted into a query. - */ - virtual void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; - virtual void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; - - /** Text serialization for the CSV format. - */ - virtual void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; - virtual void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; - - /** Text serialization intended for using in JSON format. - */ - virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; - virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; - - /** Text serialization for putting into the XML format. - */ - virtual void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const = 0; -}; - -/** Allows to customize an existing data type by representation with custom substreams. - * Customized data type will be serialized/deserialized to files with different names than base type, - * but binary and text representation will be unchanged. - * E.g it can be used for reading single subcolumns of complex types. - */ -class IDataTypeCustomStreams -{ -public: - virtual ~IDataTypeCustomStreams() = default; - - virtual void enumerateStreams( - const IDataType::StreamCallback & callback, - IDataType::SubstreamPath & path) const = 0; - - virtual void serializeBinaryBulkStatePrefix( - IDataType::SerializeBinaryBulkSettings & settings, - IDataType::SerializeBinaryBulkStatePtr & state) const = 0; - - virtual void serializeBinaryBulkStateSuffix( - IDataType::SerializeBinaryBulkSettings & settings, - IDataType::SerializeBinaryBulkStatePtr & state) const = 0; - - virtual void deserializeBinaryBulkStatePrefix( - IDataType::DeserializeBinaryBulkSettings & settings, - IDataType::DeserializeBinaryBulkStatePtr & state) const = 0; - - virtual void serializeBinaryBulkWithMultipleStreams( - const IColumn & column, - size_t offset, - size_t limit, - IDataType::SerializeBinaryBulkSettings & settings, - IDataType::SerializeBinaryBulkStatePtr & state) const = 0; - - virtual void deserializeBinaryBulkWithMultipleStreams( - ColumnPtr & column, - size_t limit, - IDataType::DeserializeBinaryBulkSettings & settings, - IDataType::DeserializeBinaryBulkStatePtr & state, - IDataType::SubstreamsCache * cache) const = 0; -}; - using DataTypeCustomNamePtr = std::unique_ptr; -using DataTypeCustomTextSerializationPtr = std::unique_ptr; -using DataTypeCustomStreamsPtr = std::unique_ptr; - /** Describe a data type customization */ struct DataTypeCustomDesc { DataTypeCustomNamePtr name; - DataTypeCustomTextSerializationPtr text_serialization; - DataTypeCustomStreamsPtr streams; + SerializationPtr serialization; DataTypeCustomDesc( DataTypeCustomNamePtr name_, - DataTypeCustomTextSerializationPtr text_serialization_ = nullptr, - DataTypeCustomStreamsPtr streams_ = nullptr) + SerializationPtr serialization_ = nullptr) : name(std::move(name_)) - , text_serialization(std::move(text_serialization_)) - , streams(std::move(streams_)) {} + , serialization(std::move(serialization_)) {} }; using DataTypeCustomDescPtr = std::unique_ptr; diff --git a/src/DataTypes/DataTypeCustomGeo.cpp b/src/DataTypes/DataTypeCustomGeo.cpp index 73d76e7e1e8..845797d3608 100644 --- a/src/DataTypes/DataTypeCustomGeo.cpp +++ b/src/DataTypes/DataTypeCustomGeo.cpp @@ -1,7 +1,5 @@ -#include #include #include -#include #include #include #include @@ -9,106 +7,20 @@ namespace DB { -namespace -{ - -class DataTypeCustomPointSerialization : public DataTypeCustomSimpleTextSerialization -{ -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique( - DataTypes({std::make_unique(), std::make_unique()}))); - return data_type; - } -}; - -class DataTypeCustomRingSerialization : public DataTypeCustomSimpleTextSerialization -{ -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPointSerialization::nestedDataType())); - return data_type; - } -}; - -class DataTypeCustomPolygonSerialization : public DataTypeCustomSimpleTextSerialization -{ -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomRingSerialization::nestedDataType())); - return data_type; - } -}; - -class DataTypeCustomMultiPolygonSerialization : public DataTypeCustomSimpleTextSerialization -{ -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPolygonSerialization::nestedDataType())); - return data_type; - } -}; - -} - void registerDataTypeDomainGeo(DataTypeFactory & factory) { // Custom type for point represented as its coordinates stored as Tuple(Float64, Float64) factory.registerSimpleDataTypeCustom("Point", [] { return std::make_pair(DataTypeFactory::instance().get("Tuple(Float64, Float64)"), - std::make_unique(std::make_unique("Point"), std::make_unique())); + std::make_unique(std::make_unique("Point"))); }); // Custom type for simple polygon without holes stored as Array(Point) factory.registerSimpleDataTypeCustom("Ring", [] { return std::make_pair(DataTypeFactory::instance().get("Array(Point)"), - std::make_unique(std::make_unique("Ring"), std::make_unique())); + std::make_unique(std::make_unique("Ring"))); }); // Custom type for polygon with holes stored as Array(Ring) @@ -116,14 +28,14 @@ void registerDataTypeDomainGeo(DataTypeFactory & factory) factory.registerSimpleDataTypeCustom("Polygon", [] { return std::make_pair(DataTypeFactory::instance().get("Array(Ring)"), - std::make_unique(std::make_unique("Polygon"), std::make_unique())); + std::make_unique(std::make_unique("Polygon"))); }); // Custom type for multiple polygons with holes stored as Array(Polygon) factory.registerSimpleDataTypeCustom("MultiPolygon", [] { return std::make_pair(DataTypeFactory::instance().get("Array(Polygon)"), - std::make_unique(std::make_unique("MultiPolygon"), std::make_unique())); + std::make_unique(std::make_unique("MultiPolygon"))); }); } diff --git a/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp b/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp index 78a1e18679d..f0eeaa21be8 100644 --- a/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp +++ b/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp @@ -1,10 +1,6 @@ -#include -#include -#include -#include +#include #include #include -#include namespace DB { @@ -15,101 +11,20 @@ namespace ErrorCodes extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING; } -namespace -{ - -class DataTypeCustomIPv4Serialization : public DataTypeCustomSimpleTextSerialization -{ -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override - { - const auto * col = checkAndGetColumn(&column); - if (!col) - { - throw Exception("IPv4 type can only serialize columns of type UInt32." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); - } - - char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'}; - char * ptr = buffer; - formatIPv4(reinterpret_cast(&col->getData()[row_num]), ptr); - - ostr.write(buffer, strlen(buffer)); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override - { - ColumnUInt32 * col = typeid_cast(&column); - if (!col) - { - throw Exception("IPv4 type can only deserialize columns of type UInt32." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); - } - - char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'}; - istr.read(buffer, sizeof(buffer) - 1); - UInt32 ipv4_value = 0; - if (!parseIPv4(buffer, reinterpret_cast(&ipv4_value))) - { - throw Exception("Invalid IPv4 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); - } - - col->insert(ipv4_value); - } -}; - -class DataTypeCustomIPv6Serialization : public DataTypeCustomSimpleTextSerialization -{ -public: - - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override - { - const auto * col = checkAndGetColumn(&column); - if (!col) - { - throw Exception("IPv6 type domain can only serialize columns of type FixedString(16)." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); - } - - char buffer[IPV6_MAX_TEXT_LENGTH + 1] = {'\0'}; - char * ptr = buffer; - formatIPv6(reinterpret_cast(col->getDataAt(row_num).data), ptr); - - ostr.write(buffer, strlen(buffer)); - } - - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override - { - ColumnFixedString * col = typeid_cast(&column); - if (!col) - { - throw Exception("IPv6 type domain can only deserialize columns of type FixedString(16)." + column.getName(), ErrorCodes::ILLEGAL_COLUMN); - } - - char buffer[IPV6_MAX_TEXT_LENGTH + 1] = {'\0'}; - istr.read(buffer, sizeof(buffer) - 1); - - std::string ipv6_value(IPV6_BINARY_LENGTH, '\0'); - if (!parseIPv6(buffer, reinterpret_cast(ipv6_value.data()))) - { - throw Exception("Invalid IPv6 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); - } - - col->insertString(ipv6_value); - } -}; - -} - void registerDataTypeDomainIPv4AndIPv6(DataTypeFactory & factory) { factory.registerSimpleDataTypeCustom("IPv4", [] { - return std::make_pair(DataTypeFactory::instance().get("UInt32"), - std::make_unique(std::make_unique("IPv4"), std::make_unique())); + auto type = DataTypeFactory::instance().get("UInt32"); + return std::make_pair(type, std::make_unique( + std::make_unique("IPv4"), std::make_unique(type->getDefaultSerialization()))); }); factory.registerSimpleDataTypeCustom("IPv6", [] { - return std::make_pair(DataTypeFactory::instance().get("FixedString(16)"), - std::make_unique(std::make_unique("IPv6"), std::make_unique())); + auto type = DataTypeFactory::instance().get("FixedString(16)"); + return std::make_pair(type, std::make_unique( + std::make_unique("IPv6"), std::make_unique(type->getDefaultSerialization()))); }); /// MySQL, MariaDB diff --git a/src/DataTypes/DataTypeCustom_fwd.h b/src/DataTypes/DataTypeCustom_fwd.h index 99c8eee9748..39c82bc4366 100644 --- a/src/DataTypes/DataTypeCustom_fwd.h +++ b/src/DataTypes/DataTypeCustom_fwd.h @@ -1,18 +1,18 @@ -#pragma once +// #pragma once -#include +// #include -namespace DB -{ +// namespace DB +// { -class IDataTypeCustomName; -class IDataTypeCustomTextSerialization; -class IDataTypeCustomStreams; -struct DataTypeCustomDesc; +// class IDataTypeCustomName; +// class IDataTypeCustomTextSerialization; +// class IDataTypeCustomStreams; +// struct DataTypeCustomDesc; -using DataTypeCustomNamePtr = std::unique_ptr; -using DataTypeCustomTextSerializationPtr = std::unique_ptr; -using DataTypeCustomStreamsPtr = std::unique_ptr; -using DataTypeCustomDescPtr = std::unique_ptr; +// using DataTypeCustomNamePtr = std::unique_ptr; +// using DataTypeCustomTextSerializationPtr = std::unique_ptr; +// using DataTypeCustomStreamsPtr = std::unique_ptr; +// using DataTypeCustomDescPtr = std::unique_ptr; -} +// } diff --git a/src/DataTypes/DataTypeDate.cpp b/src/DataTypes/DataTypeDate.cpp new file mode 100644 index 00000000000..0df2e329702 --- /dev/null +++ b/src/DataTypes/DataTypeDate.cpp @@ -0,0 +1,30 @@ +#include +#include + +#include +#include +#include +#include + +#include + + +namespace DB +{ + +bool DataTypeDate::equals(const IDataType & rhs) const +{ + return typeid(rhs) == typeid(*this); +} + +SerializationPtr DataTypeDate::doGetDefaultSerialization() const +{ + return std::make_shared(); +} + +void registerDataTypeDate(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); +} + +} diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h new file mode 100644 index 00000000000..2f17207cc07 --- /dev/null +++ b/src/DataTypes/DataTypeDate.h @@ -0,0 +1,26 @@ +#pragma once + +#include + + +namespace DB +{ + +class DataTypeDate final : public DataTypeNumberBase +{ +public: + static constexpr auto family_name = "Date"; + + TypeIndex getTypeId() const override { return TypeIndex::Date; } + const char * getFamilyName() const override { return family_name; } + + bool canBeUsedAsVersion() const override { return true; } + bool canBeInsideNullable() const override { return true; } + + bool equals(const IDataType & rhs) const override; + +protected: + SerializationPtr doGetDefaultSerialization() const override; +}; + +} diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp new file mode 100644 index 00000000000..0bcb3d75616 --- /dev/null +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +TimezoneMixin::TimezoneMixin(const String & time_zone_name) + : has_explicit_time_zone(!time_zone_name.empty()), + time_zone(DateLUT::instance(time_zone_name)), + utc_time_zone(DateLUT::instance("UTC")) +{} + +DataTypeDateTime::DataTypeDateTime(const String & time_zone_name) + : TimezoneMixin(time_zone_name) +{ +} + +DataTypeDateTime::DataTypeDateTime(const TimezoneMixin & time_zone_) + : TimezoneMixin(time_zone_) +{} + +String DataTypeDateTime::doGetName() const +{ + if (!has_explicit_time_zone) + return "DateTime"; + + WriteBufferFromOwnString out; + out << "DateTime(" << quote << time_zone.getTimeZone() << ")"; + return out.str(); +} + +bool DataTypeDateTime::equals(const IDataType & rhs) const +{ + /// DateTime with different timezones are equal, because: + /// "all types with different time zones are equivalent and may be used interchangingly." + return typeid(rhs) == typeid(*this); +} + +SerializationPtr DataTypeDateTime::doGetDefaultSerialization() const +{ + return std::make_shared(time_zone, utc_time_zone); +} + +} diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h new file mode 100644 index 00000000000..84df8b21813 --- /dev/null +++ b/src/DataTypes/DataTypeDateTime.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +class DateLUTImpl; + +namespace DB +{ + +/** Mixin-class that manages timezone info for timezone-aware DateTime implementations + * + * Must be used as a (second) base for class implementing IDateType-interface. + */ +class TimezoneMixin +{ +public: + explicit TimezoneMixin(const String & time_zone_name = ""); + TimezoneMixin(const TimezoneMixin &) = default; + + const DateLUTImpl & getTimeZone() const { return time_zone; } + +protected: + bool has_explicit_time_zone; + const DateLUTImpl & time_zone; + const DateLUTImpl & utc_time_zone; +}; + +/** DateTime stores time as unix timestamp. + * The value itself is independent of time zone. + * + * In binary format it is represented as unix timestamp. + * In text format it is serialized to and parsed from YYYY-MM-DD hh:mm:ss format. + * The text format is dependent of time zone. + * + * To cast from/to text format, time zone may be specified explicitly or implicit time zone may be used. + * + * Time zone may be specified explicitly as type parameter, example: DateTime('Europe/Moscow'). + * As it does not affect the internal representation of values, + * all types with different time zones are equivalent and may be used interchangingly. + * Time zone only affects parsing and displaying in text formats. + * + * If time zone is not specified (example: DateTime without parameter), then default time zone is used. + * Default time zone is server time zone, if server is doing transformations + * and if client is doing transformations, unless 'use_client_time_zone' setting is passed to client; + * Server time zone is the time zone specified in 'timezone' parameter in configuration file, + * or system time zone at the moment of server startup. + */ +class DataTypeDateTime final : public DataTypeNumberBase, public TimezoneMixin +{ +public: + explicit DataTypeDateTime(const String & time_zone_name = ""); + explicit DataTypeDateTime(const TimezoneMixin & time_zone); + + static constexpr auto family_name = "DateTime"; + + const char * getFamilyName() const override { return family_name; } + String doGetName() const override; + TypeIndex getTypeId() const override { return TypeIndex::DateTime; } + + bool canBeUsedAsVersion() const override { return true; } + bool canBeInsideNullable() const override { return true; } + + bool equals(const IDataType & rhs) const override; + + SerializationPtr doGetDefaultSerialization() const override; +}; + +} + diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/DataTypeDateTime64.cpp new file mode 100644 index 00000000000..eaec585b6b4 --- /dev/null +++ b/src/DataTypes/DataTypeDateTime64.cpp @@ -0,0 +1,71 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +static constexpr UInt32 max_scale = 9; + +DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name) + : DataTypeDecimalBase(DecimalUtils::max_precision, scale_), + TimezoneMixin(time_zone_name) +{ + if (scale > max_scale) + throw Exception("Scale " + std::to_string(scale) + " is too large for DateTime64. Maximum is up to nanoseconds (9).", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); +} + +DataTypeDateTime64::DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info) + : DataTypeDecimalBase(DecimalUtils::max_precision, scale_), + TimezoneMixin(time_zone_info) +{ + if (scale > max_scale) + throw Exception("Scale " + std::to_string(scale) + " is too large for DateTime64. Maximum is up to nanoseconds (9).", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); +} + +std::string DataTypeDateTime64::doGetName() const +{ + if (!has_explicit_time_zone) + return std::string(getFamilyName()) + "(" + std::to_string(this->scale) + ")"; + + WriteBufferFromOwnString out; + out << "DateTime64(" << this->scale << ", " << quote << time_zone.getTimeZone() << ")"; + return out.str(); +} + +bool DataTypeDateTime64::equals(const IDataType & rhs) const +{ + if (const auto * ptype = typeid_cast(&rhs)) + return this->scale == ptype->getScale(); + return false; +} + +SerializationPtr DataTypeDateTime64::doGetDefaultSerialization() const +{ + return std::make_shared(time_zone, utc_time_zone, scale); +} + +} diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h new file mode 100644 index 00000000000..ba03c40f14e --- /dev/null +++ b/src/DataTypes/DataTypeDateTime64.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include + +class DateLUTImpl; + +namespace DB +{ + +/** DateTime64 is same as DateTime, but it stores values as Int64 and has configurable sub-second part. + * + * `scale` determines number of decimal places for sub-second part of the DateTime64. + */ +class DataTypeDateTime64 final : public DataTypeDecimalBase, public TimezoneMixin +{ +public: + using Base = DataTypeDecimalBase; + static constexpr UInt8 default_scale = 3; + + static constexpr auto family_name = "DateTime64"; + static constexpr auto type_id = TypeIndex::DateTime64; + + explicit DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name = ""); + + // reuse timezone from other DateTime/DateTime64 + DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info); + + const char * getFamilyName() const override { return family_name; } + std::string doGetName() const override; + TypeIndex getTypeId() const override { return type_id; } + + bool equals(const IDataType & rhs) const override; + + bool canBePromoted() const override { return false; } + +protected: + SerializationPtr doGetDefaultSerialization() const override; +}; + +/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. + * + * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, + * invokes Transform::execute() with: + * * whole part of DateTime64 value, discarding fractional part. + * * DateTime64 value and scale factor. + * + * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * and should implement static (or const) function with following signatures: + * R execute(UInt32 whole_value, ... , const TimeZoneImpl &) + * OR + * R execute(DateTime64 value, Int64 scale_factor, ... , const TimeZoneImpl &) + * + * Where R and T could be arbitrary types. +*/ +template +class TransformDateTime64 : public Transform +{ +private: + // Detect if Transform::execute is const or static method + // with signature defined by template args (ignoring result type). + template + struct TransformHasExecuteOverload : std::false_type {}; + + template + struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> + : std::true_type {}; + + template + static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; + +public: + static constexpr auto name = Transform::name; + + using Transform::execute; + + // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. + TransformDateTime64(UInt32 scale_ = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) + {} + + template + inline auto execute(const DateTime64 & t, Args && ... args) const + { + const auto transform = static_cast(this); + + if constexpr (TransformHasExecuteOverload_v) + { + return transform->execute(t, scale_multiplier, std::forward(args)...); + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return transform->execute(static_cast(components.whole), std::forward(args)...); + } + } + +private: + DateTime64::NativeType scale_multiplier = 1; +}; + +} + diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/DataTypeDecimalBase.cpp new file mode 100644 index 00000000000..5f64fca6704 --- /dev/null +++ b/src/DataTypes/DataTypeDecimalBase.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ +} + + +bool decimalCheckComparisonOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; } +bool decimalCheckArithmeticOverflow(const Context & context) { return context.getSettingsRef().decimal_check_overflow; } + +template +Field DataTypeDecimalBase::getDefault() const +{ + return DecimalField(T(0), scale); +} + +template +MutableColumnPtr DataTypeDecimalBase::createColumn() const +{ + return ColumnType::create(0, scale); +} + +template +T DataTypeDecimalBase::getScaleMultiplier(UInt32 scale_) +{ + return DecimalUtils::scaleMultiplier(scale_); +} + + +/// Explicit template instantiations. +template class DataTypeDecimalBase; +template class DataTypeDecimalBase; +template class DataTypeDecimalBase; +template class DataTypeDecimalBase; +template class DataTypeDecimalBase; + +} diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h new file mode 100644 index 00000000000..85cb9fa8363 --- /dev/null +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -0,0 +1,206 @@ +#pragma once +#include + +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +class Context; +bool decimalCheckComparisonOverflow(const Context & context); +bool decimalCheckArithmeticOverflow(const Context & context); + +inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type) +{ + switch (int_type) + { + case TypeIndex::Int8: [[fallthrough]]; + case TypeIndex::UInt8: + return 3; + case TypeIndex::Int16: [[fallthrough]]; + case TypeIndex::UInt16: + return 5; + case TypeIndex::Int32: [[fallthrough]]; + case TypeIndex::UInt32: + return 10; + case TypeIndex::Int64: + return 19; + case TypeIndex::UInt64: + return 20; + default: + break; + } + return 0; +} + +/// Base class for decimals, like Decimal(P, S), where P is precision, S is scale. +/// Maximum precisions for underlying types are: +/// Int32 9 +/// Int64 18 +/// Int128 38 +/// Int256 76 +/// Operation between two decimals leads to Decimal(P, S), where +/// P is one of (9, 18, 38, 76); equals to the maximum precision for the biggest underlying type of operands. +/// S is maximum scale of operands. The allowed valuas are [0, precision] +template +class DataTypeDecimalBase : public IDataType +{ + static_assert(IsDecimalNumber); + +public: + using FieldType = T; + using ColumnType = ColumnDecimal; + + static constexpr bool is_parametric = true; + + static constexpr size_t maxPrecision() { return DecimalUtils::max_precision; } + + DataTypeDecimalBase(UInt32 precision_, UInt32 scale_) + : precision(precision_), + scale(scale_) + { + if (unlikely(precision < 1 || precision > maxPrecision())) + throw Exception("Precision " + std::to_string(precision) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + if (unlikely(scale > maxPrecision())) + throw Exception("Scale " + std::to_string(scale) + " is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + + TypeIndex getTypeId() const override { return TypeId::value; } + + Field getDefault() const override; + MutableColumnPtr createColumn() const override; + + bool isParametric() const override { return true; } + bool haveSubtypes() const override { return false; } + bool shouldAlignRightInPrettyFormats() const override { return true; } + bool textCanContainOnlyValidUTF8() const override { return true; } + bool isComparable() const override { return true; } + bool isValueRepresentedByNumber() const override { return true; } + bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; } + bool haveMaximumSizeOfValue() const override { return true; } + size_t getSizeOfValueInMemory() const override { return sizeof(T); } + + bool isSummable() const override { return true; } + bool canBeUsedInBooleanContext() const override { return true; } + bool canBeInsideNullable() const override { return true; } + + /// Decimal specific + + UInt32 getPrecision() const { return precision; } + UInt32 getScale() const { return scale; } + T getScaleMultiplier() const { return getScaleMultiplier(scale); } + + T wholePart(T x) const + { + return DecimalUtils::getWholePart(x, scale); + } + + T fractionalPart(T x) const + { + return DecimalUtils::getFractionalPart(x, scale); + } + + T maxWholeValue() const { return getScaleMultiplier(precision - scale) - T(1); } + + template + bool canStoreWhole(U x) const + { + static_assert(std::is_signed_v); + T max = maxWholeValue(); + if constexpr (std::is_signed_v) + return -max <= x && x <= max; + else + return x <= static_cast>(max.value); + } + + /// @returns multiplier for U to become T with correct scale + template + T scaleFactorFor(const DataTypeDecimalBase & x, bool) const + { + if (getScale() < x.getScale()) + throw Exception("Decimal result's scale is less than argument's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + UInt32 scale_delta = getScale() - x.getScale(); /// scale_delta >= 0 + return getScaleMultiplier(scale_delta); + } + + template + T scaleFactorFor(const DataTypeNumber & , bool is_multiply_or_divisor) const + { + if (is_multiply_or_divisor) + return T(1); + return getScaleMultiplier(); + } + + static T getScaleMultiplier(UInt32 scale); + + inline DecimalUtils::DataTypeDecimalTrait getTrait() const + { + return {precision, scale}; + } + +protected: + const UInt32 precision; + const UInt32 scale; +}; + + +template +inline const DataTypeDecimalBase * checkDecimalBase(const IDataType & data_type) +{ + if (isColumnedAsDecimalT(data_type)) + return static_cast *>(&data_type); + + return nullptr; +} + +template typename DecimalType> +inline auto decimalResultType(const DecimalType & tx, const DecimalType & ty) +{ + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); +} + +template typename DecimalType> +inline const DecimalType decimalResultType(const DecimalType & tx, const DataTypeNumber & ty) +{ + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); +} + +template typename DecimalType> +inline const DecimalType decimalResultType(const DataTypeNumber & tx, const DecimalType & ty) +{ + const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); + return DecimalType(result_trait.precision, result_trait.scale); +} + +template