From 132bbce29cf26f2e93de852e6ddce9f2e3f0023d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 18 Feb 2022 17:42:32 +0300 Subject: [PATCH 1/4] Add ability to get SELECT query from TableFunctionView Signed-off-by: Azat Khuzhin --- src/TableFunctions/TableFunctionView.cpp | 6 ++++++ src/TableFunctions/TableFunctionView.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 2cab8aeca25..e9fcbb219a3 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -15,6 +15,12 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } + +const ASTSelectWithUnionQuery & TableFunctionView::getSelectQuery() const +{ + return *create.select; +} + void TableFunctionView::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/) { const auto * function = ast_function->as(); diff --git a/src/TableFunctions/TableFunctionView.h b/src/TableFunctions/TableFunctionView.h index c20b45e7546..4afb049e738 100644 --- a/src/TableFunctions/TableFunctionView.h +++ b/src/TableFunctions/TableFunctionView.h @@ -16,6 +16,9 @@ class TableFunctionView : public ITableFunction public: static constexpr auto name = "view"; std::string getName() const override { return name; } + + const ASTSelectWithUnionQuery & getSelectQuery() const; + private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const String & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "View"; } From fd3f7347f3e5f9dbc4f1f7279be8687ff85a0560 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 19 Feb 2022 10:48:06 +0300 Subject: [PATCH 2/4] Remove unused DBMS_COMMON_LIBRARIES Fixes: 4f8438bb346 ("cmake: do not allow internal libstdc++ usage") Signed-off-by: Azat Khuzhin --- src/CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b99ffd7ee18..0a831950d5e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -253,18 +253,16 @@ if (TARGET ch_contrib::nuraft) add_object_library(clickhouse_coordination Coordination) endif() -set (DBMS_COMMON_LIBRARIES) - if (USE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) add_library (dbms STATIC ${dbms_headers} ${dbms_sources}) - target_link_libraries (dbms PRIVATE ch_contrib::libdivide ${DBMS_COMMON_LIBRARIES}) + target_link_libraries (dbms PRIVATE ch_contrib::libdivide) if (TARGET ch_contrib::jemalloc) target_link_libraries (dbms PRIVATE ch_contrib::jemalloc) endif() set (all_modules dbms) else() add_library (dbms SHARED ${dbms_headers} ${dbms_sources}) - target_link_libraries (dbms PUBLIC ${all_modules} ${DBMS_COMMON_LIBRARIES}) + target_link_libraries (dbms PUBLIC ${all_modules}) target_link_libraries (clickhouse_interpreters PRIVATE ch_contrib::libdivide) if (TARGET ch_contrib::jemalloc) target_link_libraries (clickhouse_interpreters PRIVATE ch_contrib::jemalloc) From 75da778d10fd005703523ddd9837617803be0d69 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 19 Feb 2022 11:02:45 +0300 Subject: [PATCH 3/4] Tiny cmake refactoring Signed-off-by: Azat Khuzhin --- src/CMakeLists.txt | 22 +++++++++++++++++----- src/TableFunctions/CMakeLists.txt | 16 +++++++++------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a831950d5e..ebcd027ed2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -144,7 +144,6 @@ list (APPEND dbms_sources AggregateFunctions/AggregateFunctionState.cpp AggregateFunctions/AggregateFunctionCount.cpp AggregateFunctions/parseAggregateFunctionParameters.cpp) - list (APPEND dbms_headers AggregateFunctions/IAggregateFunction.h AggregateFunctions/IAggregateFunctionCombinator.h @@ -155,10 +154,23 @@ list (APPEND dbms_headers AggregateFunctions/FactoryHelpers.h AggregateFunctions/parseAggregateFunctionParameters.h) -list (APPEND dbms_sources TableFunctions/ITableFunction.cpp TableFunctions/TableFunctionFactory.cpp) -list (APPEND dbms_headers TableFunctions/ITableFunction.h TableFunctions/TableFunctionFactory.h) -list (APPEND dbms_sources Dictionaries/DictionaryFactory.cpp Dictionaries/DictionarySourceFactory.cpp Dictionaries/DictionaryStructure.cpp Dictionaries/getDictionaryConfigurationFromAST.cpp) -list (APPEND dbms_headers Dictionaries/DictionaryFactory.h Dictionaries/DictionarySourceFactory.h Dictionaries/DictionaryStructure.h Dictionaries/getDictionaryConfigurationFromAST.h) +list (APPEND dbms_sources + TableFunctions/ITableFunction.cpp + TableFunctions/TableFunctionFactory.cpp) +list (APPEND dbms_headers + TableFunctions/ITableFunction.h + TableFunctions/TableFunctionFactory.h) + +list (APPEND dbms_sources + Dictionaries/DictionaryFactory.cpp + Dictionaries/DictionarySourceFactory.cpp + Dictionaries/DictionaryStructure.cpp + Dictionaries/getDictionaryConfigurationFromAST.cpp) +list (APPEND dbms_headers + Dictionaries/DictionaryFactory.h + Dictionaries/DictionarySourceFactory.h + Dictionaries/DictionaryStructure.h + Dictionaries/getDictionaryConfigurationFromAST.h) if (NOT ENABLE_SSL) list (REMOVE_ITEM clickhouse_common_io_sources Common/OpenSSLHelpers.cpp) diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index c9948a4b131..21d329667a5 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -4,14 +4,16 @@ if (TARGET ch_contrib::hivemetastore) add_headers_and_sources(clickhouse_table_functions Hive) endif () -list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp TableFunctionFactory.cpp) -list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h) +list(REMOVE_ITEM clickhouse_table_functions_sources + ITableFunction.cpp + TableFunctionFactory.cpp) +list(REMOVE_ITEM clickhouse_table_functions_headers + ITableFunction.h + TableFunctionFactory.h) add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) + +target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) if (TARGET ch_contrib::hivemetastore) - target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms ch_contrib::hivemetastore ch_contrib::hdfs) -else () - target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) + target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::hivemetastore ch_contrib::hdfs) endif () - - From 4843e210c322b07fcd3a899afa35cf7a31109441 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 18 Feb 2022 17:42:48 +0300 Subject: [PATCH 4/4] Support view() for parallel_distributed_insert_select Signed-off-by: Azat Khuzhin --- src/CMakeLists.txt | 2 ++ src/Storages/StorageDistributed.cpp | 26 ++++++++++---- src/TableFunctions/CMakeLists.txt | 2 ++ ...l_distributed_insert_select_view.reference | 4 +++ ...parallel_distributed_insert_select_view.sh | 35 +++++++++++++++++++ 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference create mode 100755 tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ebcd027ed2b..e18914740ff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -156,9 +156,11 @@ list (APPEND dbms_headers list (APPEND dbms_sources TableFunctions/ITableFunction.cpp + TableFunctions/TableFunctionView.cpp TableFunctions/TableFunctionFactory.cpp) list (APPEND dbms_headers TableFunctions/ITableFunction.h + TableFunctions/TableFunctionView.h TableFunctions/TableFunctionFactory.h) list (APPEND dbms_sources diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index fcbf22bbd33..5bfb3b4ce45 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -56,6 +56,8 @@ #include #include #include +#include +#include #include #include @@ -723,15 +725,27 @@ QueryPipelineBuilderPtr StorageDistributed::distributedWrite(const ASTInsertQuer storage_src = std::dynamic_pointer_cast(joined_tables.getLeftTableStorage()); if (storage_src) { - const auto select_with_union_query = std::make_shared(); - select_with_union_query->list_of_selects = std::make_shared(); + /// Unwrap view() function. + if (storage_src->remote_table_function_ptr) + { + const TableFunctionPtr src_table_function = + TableFunctionFactory::instance().get(storage_src->remote_table_function_ptr, local_context); + const TableFunctionView * view_function = + assert_cast(src_table_function.get()); + new_query->select = view_function->getSelectQuery().clone(); + } + else + { + const auto select_with_union_query = std::make_shared(); + select_with_union_query->list_of_selects = std::make_shared(); - auto new_select_query = std::dynamic_pointer_cast(select_query->clone()); - select_with_union_query->list_of_selects->children.push_back(new_select_query); + auto new_select_query = std::dynamic_pointer_cast(select_query->clone()); + select_with_union_query->list_of_selects->children.push_back(new_select_query); - new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); + new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); - new_query->select = select_with_union_query; + new_query->select = select_with_union_query; + } } } } diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 21d329667a5..c58f93e310a 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -6,9 +6,11 @@ endif () list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp + TableFunctionView.cpp TableFunctionFactory.cpp) list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h + TableFunctionView.h TableFunctionFactory.h) add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) diff --git a/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference new file mode 100644 index 00000000000..98fb6a68656 --- /dev/null +++ b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh new file mode 100755 index 00000000000..376a49fd820 --- /dev/null +++ b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# NOTE: sh test is required since view() does not have current database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " +drop table if exists dst_02225; +drop table if exists src_02225; +create table dst_02225 (key Int) engine=Memory(); +create table src_02225 (key Int) engine=Memory(); +insert into src_02225 values (1); +" + +$CLICKHOUSE_CLIENT --param_database=$CLICKHOUSE_DATABASE -nm -q " +truncate table dst_02225; +insert into function remote('127.{1,2}', currentDatabase(), dst_02225, key) +select * from remote('127.{1,2}', view(select * from {database:Identifier}.src_02225), key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02225; + +-- w/o sharding key +truncate table dst_02225; +insert into function remote('127.{1,2}', currentDatabase(), dst_02225, key) +select * from remote('127.{1,2}', view(select * from {database:Identifier}.src_02225)) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02225; +" + +$CLICKHOUSE_CLIENT -nm -q " +drop table src_02225; +drop table dst_02225; +"