From 14a968b08a61592317ac9390cf726943a54b5b26 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 30 Aug 2024 02:21:54 +0000 Subject: [PATCH 01/14] Support create a new table with CLONE AS --- src/Interpreters/InterpreterCreateQuery.cpp | 75 ++++++++++++++++++--- src/Parsers/ASTCreateQuery.cpp | 11 +++ src/Parsers/ASTCreateQuery.h | 1 + src/Parsers/CommonParsers.h | 1 + src/Parsers/ParserCreateQuery.cpp | 9 ++- 5 files changed, 87 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 80cb0510b35..f0ab8da5087 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -5,17 +5,19 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -826,6 +828,10 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti { /// Only MergeTree support TTL properties.columns.resetColumnTTLs(); + if (create.is_clone_as) + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support 'CLONE AS' with tables of MergeTree family"); + } } properties.constraints = as_storage_metadata->getConstraints(); @@ -1933,6 +1939,57 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) /* async_isnert */ false).execute(); } + /// If the query is a CREATE .. CLONE AS , insert the data into the table. + if (create.is_clone_as && !as_table_saved.empty() && !create.is_create_empty && !create.is_ordinary_view && !create.is_live_view + && (!(create.is_materialized_view || create.is_window_view) || create.is_populate)) + { + String as_database_name = getContext()->resolveDatabase(create.as_database); + StorageID as_table_id = {as_database_name, as_table_saved}; + StoragePtr as_table = DatabaseCatalog::instance().tryGetTable(as_table_id, getContext()); + if (!as_table) + { + return {}; + } + + auto merge_tree_table = std::dynamic_pointer_cast(as_table); + if (!merge_tree_table) + { + return {}; + } + + auto command_list = std::make_shared(); + for (const auto & partition_id : merge_tree_table->getAllPartitionIds()) + { + auto partition = std::make_shared(); + partition->all = false; + partition->setPartitionID(std::make_shared(partition_id)); + + auto command = std::make_shared(); + command->replace = false; + + command->type = ASTAlterCommand::REPLACE_PARTITION; + command->partition = command->children.emplace_back(std::move(partition)).get(); + + command->from_database = as_database_name; + command->from_table = as_table_saved; + + command->to_database = create.getDatabase(); + command->to_table = create.getTable(); + + command_list->children.push_back(command); + } + + auto query = std::make_shared(); + query->database = create.database; + query->table = create.table; + query->uuid = create.uuid; + auto * alter = query->as(); + + alter->alter_object = ASTAlterQuery::AlterObjectType::TABLE; + alter->set(alter->command_list, command_list); + return InterpreterAlterQuery(query, getContext()).execute(); + } + return {}; } diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index d7f5b8f9702..66422efe660 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -425,9 +425,19 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : ""); }; + bool should_add_clone = is_clone_as; + auto add_clone_if_needed = [&] + { + if (!should_add_clone) + return; + should_add_clone = false; + settings.ostr << (settings.hilite ? hilite_keyword : "") << " CLONE" << (settings.hilite ? hilite_none : ""); + }; + if (!as_table.empty()) { add_empty_if_needed(); + add_clone_if_needed(); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table); @@ -446,6 +456,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat } add_empty_if_needed(); + add_clone_if_needed(); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); as_table_function->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 6be0fa78903..813e56eaf02 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -100,6 +100,7 @@ public: bool is_time_series_table{false}; /// CREATE TABLE ... ENGINE=TimeSeries() ... bool is_populate{false}; bool is_create_empty{false}; /// CREATE TABLE ... EMPTY AS SELECT ... + bool is_clone_as{false}; /// CREATE TABLE ... CLONE AS ... bool replace_view{false}; /// CREATE OR REPLACE VIEW bool has_uuid{false}; // CREATE TABLE x UUID '...' diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index ab0e70eb0e5..59cb9499ef1 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -84,6 +84,7 @@ namespace DB MR_MACROS(CLEAR_INDEX, "CLEAR INDEX") \ MR_MACROS(CLEAR_PROJECTION, "CLEAR PROJECTION") \ MR_MACROS(CLEAR_STATISTICS, "CLEAR STATISTICS") \ + MR_MACROS(CLONE_AS, "CLONE AS") \ MR_MACROS(CLUSTER, "CLUSTER") \ MR_MACROS(CLUSTERS, "CLUSTERS") \ MR_MACROS(CN, "CN") \ diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 31dc2075db4..8c5b926ef71 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -675,6 +675,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe bool if_not_exists = false; bool is_temporary = false; bool is_create_empty = false; + bool is_clone_as = false; if (s_create.ignore(pos, expected)) { @@ -759,13 +760,18 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; }; - auto need_parse_as_select = [&is_create_empty, &pos, &expected]() + auto need_parse_as_select = [&is_create_empty, &is_clone_as, &pos, &expected]() { if (ParserKeyword{Keyword::EMPTY_AS}.ignore(pos, expected)) { is_create_empty = true; return true; } + if (ParserKeyword{Keyword::CLONE_AS}.ignore(pos, expected)) + { + is_clone_as = true; + return true; + } return ParserKeyword{Keyword::AS}.ignore(pos, expected); }; @@ -893,6 +899,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe query->set(query->select, select); query->set(query->targets, targets); query->is_create_empty = is_create_empty; + query->is_clone_as = is_clone_as; if (from_path) query->attach_from_path = from_path->as().value.safeGet(); From 809a73963663f9e74b946e86a3529eb6fe33d3b9 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 30 Aug 2024 08:12:25 +0000 Subject: [PATCH 02/14] Add stateless test for CREATE with CLONE AS --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +- .../03231_create_with_clone_as.reference | 8 +++ .../0_stateless/03231_create_with_clone_as.sh | 55 +++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03231_create_with_clone_as.reference create mode 100755 tests/queries/0_stateless/03231_create_with_clone_as.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f0ab8da5087..1006340544b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -830,7 +830,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti properties.columns.resetColumnTTLs(); if (create.is_clone_as) { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support 'CLONE AS' with tables of MergeTree family"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); } } @@ -1939,7 +1939,7 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) /* async_isnert */ false).execute(); } - /// If the query is a CREATE .. CLONE AS
, insert the data into the table. + /// If the query is a CREATE TABLE .. CLONE AS ..., attach all partitions of the source table to the newly created table. if (create.is_clone_as && !as_table_saved.empty() && !create.is_create_empty && !create.is_ordinary_view && !create.is_live_view && (!(create.is_materialized_view || create.is_window_view) || create.is_populate)) { diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference new file mode 100644 index 00000000000..516a04689ee --- /dev/null +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -0,0 +1,8 @@ +1 +1 +CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 +2 +CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 +2 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh new file mode 100755 index 00000000000..d8ac60ea785 --- /dev/null +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +clickhouse-client -q "DROP TABLE IF EXISTS foo_memory" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_memory" +clickhouse-client -q "DROP TABLE IF EXISTS foo_file" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_file" +clickhouse-client -q "DROP TABLE IF EXISTS foo_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" + +# CLONE AS with a table of Memory engine +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_memory (x Int8) ENGINE=Memory" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_memory VALUES (1), (2)" + +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="CREATE TABLE clone_as_foo_memory CLONE AS foo_memory" 2>&1)" \ + | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' + + +# CLONE AS with a table of File engine +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_file (x Int8) ENGINE=File(TabSeparated)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_file VALUES (1), (2)" + +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="CREATE TABLE clone_as_foo_file CLONE AS foo_file" 2>&1)" \ + | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' + +# CLONE AS with a table of MergeTree engine +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_merge_tree (x Int8) ENGINE=MergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_merge_tree VALUES (1), (2)" + +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SHOW CREATE TABLE clone_as_foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SELECT * FROM foo_merge_tree" + +# CLONE AS with a table of ReplacingMergeTree engine +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_replacing_merge_tree (x Int8) ENGINE=ReplacingMergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_replacing_merge_tree VALUES (1), (2)" + +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SELECT * FROM clone_as_foo_replacing_merge_tree" + + +clickhouse-client -q "DROP TABLE IF EXISTS foo_memory" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_memory" +clickhouse-client -q "DROP TABLE IF EXISTS foo_file" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_file" +clickhouse-client -q "DROP TABLE IF EXISTS foo_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" +clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" \ No newline at end of file From 8e1c823b3560403cfede4fdcad1f76cce13bfcf7 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 30 Aug 2024 14:44:15 +0000 Subject: [PATCH 03/14] Update test case: show create table and values of the original tables. --- .../03231_create_with_clone_as.reference | 8 +++ .../0_stateless/03231_create_with_clone_as.sh | 70 ++++++++++--------- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index 516a04689ee..851dfc85c00 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -1,8 +1,16 @@ +CREATE TABLE default.foo_memory\n(\n `x` Int8\n)\nENGINE = Memory 1 +CREATE TABLE default.foo_file\n(\n `x` Int8\n)\nENGINE = File(\'TabSeparated\') 1 +CREATE TABLE default.foo_merge_tree\n(\n `x` Int8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 +2 CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 2 +CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 +2 CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 2 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index d8ac60ea785..a51ba3ca6b9 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -4,52 +4,58 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -clickhouse-client -q "DROP TABLE IF EXISTS foo_memory" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_memory" -clickhouse-client -q "DROP TABLE IF EXISTS foo_file" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_file" -clickhouse-client -q "DROP TABLE IF EXISTS foo_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_file" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_file" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" # CLONE AS with a table of Memory engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_memory (x Int8) ENGINE=Memory" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_memory VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_memory (x Int8) ENGINE=Memory" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_memory" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_memory VALUES (1), (2)" -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="CREATE TABLE clone_as_foo_memory CLONE AS foo_memory" 2>&1)" \ +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_memory CLONE AS foo_memory" 2>&1)" \ | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' # CLONE AS with a table of File engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_file (x Int8) ENGINE=File(TabSeparated)" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_file VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_file (x Int8) ENGINE=File(TabSeparated)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_file" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_file VALUES (1), (2)" -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="CREATE TABLE clone_as_foo_file CLONE AS foo_file" 2>&1)" \ +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_file CLONE AS foo_file" 2>&1)" \ | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' # CLONE AS with a table of MergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_merge_tree (x Int8) ENGINE=MergeTree PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_merge_tree (x Int8) ENGINE=MergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SHOW CREATE TABLE clone_as_foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SELECT * FROM foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" # CLONE AS with a table of ReplacingMergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE foo_replacing_merge_tree (x Int8) ENGINE=ReplacingMergeTree PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="INSERT INTO foo_replacing_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replacing_merge_tree (x Int8) ENGINE=ReplacingMergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replacing_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="SELECT * FROM clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS foo_memory" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_memory" -clickhouse-client -q "DROP TABLE IF EXISTS foo_file" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_file" -clickhouse-client -q "DROP TABLE IF EXISTS foo_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" -clickhouse-client -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" \ No newline at end of file +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_file" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_file" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" \ No newline at end of file From 63b57c09877ed61d5a5343a3aa5b3059bdbc92b2 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Mon, 2 Sep 2024 01:03:38 +0000 Subject: [PATCH 04/14] Show mutation of stable in the test --- .../03231_create_with_clone_as.reference | 32 +++++++++++-------- .../0_stateless/03231_create_with_clone_as.sh | 22 ++++++++----- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index 851dfc85c00..18ae27f66e6 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -1,16 +1,20 @@ -CREATE TABLE default.foo_memory\n(\n `x` Int8\n)\nENGINE = Memory +CREATE TABLE default.foo_memory\n(\n `x` Int8,\n `y` String\n)\nENGINE = Memory 1 -CREATE TABLE default.foo_file\n(\n `x` Int8\n)\nENGINE = File(\'TabSeparated\') +CREATE TABLE default.foo_file\n(\n `x` Int8,\n `y` String\n)\nENGINE = File(\'TabSeparated\') 1 -CREATE TABLE default.foo_merge_tree\n(\n `x` Int8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 -1 -2 -CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 -1 -2 -CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 -1 -2 -CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 -1 -2 +CREATE TABLE default.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +1 a +2 b +CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +1 a +2 b diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index a51ba3ca6b9..b0e323f426a 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -14,40 +14,46 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" # CLONE AS with a table of Memory engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_memory (x Int8) ENGINE=Memory" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_memory" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_memory VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_memory VALUES (1, 'a'), (2, 'b')" echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_memory CLONE AS foo_memory" 2>&1)" \ | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' # CLONE AS with a table of File engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_file (x Int8) ENGINE=File(TabSeparated)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_file (x Int8, y String) ENGINE=File(TabSeparated)" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_file" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_file VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_file VALUES (1, 'a'), (2, 'b')" echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_file CLONE AS foo_file" 2>&1)" \ | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' # CLONE AS with a table of MergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_merge_tree (x Int8) ENGINE=MergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b')" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_merge_tree" # CLONE AS with a table of ReplacingMergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replacing_merge_tree (x Int8) ENGINE=ReplacingMergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replacing_merge_tree VALUES (1), (2)" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replacing_merge_tree VALUES (1, 'a'), (2, 'b')" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \ + FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'foo_replacing_merge_tree' ORDER BY mutation_id" +${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \ + FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" From 28a77f2ef010c444ba7fe587639d278177f8a6e2 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Mon, 2 Sep 2024 03:40:19 +0000 Subject: [PATCH 05/14] Separate select result in test case --- .../0_stateless/03231_create_with_clone_as.reference | 4 ++++ tests/queries/0_stateless/03231_create_with_clone_as.sh | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index 18ae27f66e6..e57d65429cb 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -6,15 +6,19 @@ CREATE TABLE default.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE 1 a 2 b CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_merge_tree 1 a 2 b +from clone_as_foo_merge_tree 1 a 2 b CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 a 2 b CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_replacing_merge_tree 1 a 2 b +from clone_as_foo_replacing_merge_tree 1 a 2 b diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index b0e323f426a..5729b398497 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -38,7 +38,9 @@ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_merge_tree" +echo "from foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" +echo "from clone_as_foo_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_merge_tree" # CLONE AS with a table of ReplacingMergeTree engine @@ -49,11 +51,9 @@ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \ - FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'foo_replacing_merge_tree' ORDER BY mutation_id" -${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done \ - FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" +echo "from foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" +echo "from clone_as_foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" From 4455d354c79644d3eea755e43e35f3606421f257 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Mon, 2 Sep 2024 07:53:36 +0000 Subject: [PATCH 06/14] Show muatations of clone_as_foo_replacing_merge_tree --- tests/queries/0_stateless/03231_create_with_clone_as.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index 5729b398497..58fac744ec0 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -54,8 +54,9 @@ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_f echo "from foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" echo "from clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" - +${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree FINAL" +${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" From 782353d8316a5f4cca86eb98034dc48a6de7518f Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Tue, 3 Sep 2024 08:03:13 +0000 Subject: [PATCH 07/14] Use ATTACH PARTITION ALL to copy data from the source table --- src/Interpreters/InterpreterCreateQuery.cpp | 33 ++++++++----------- .../0_stateless/03231_create_with_clone_as.sh | 2 -- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 1006340544b..6506e99a14d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1957,27 +1957,20 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) return {}; } + auto partition = std::make_shared(); + partition->all = true; + + auto command = std::make_shared(); + command->replace = false; + command->type = ASTAlterCommand::REPLACE_PARTITION; + command->partition = command->children.emplace_back(std::move(partition)).get(); + command->from_database = as_database_name; + command->from_table = as_table_saved; + command->to_database = create.getDatabase(); + command->to_table = create.getTable(); + auto command_list = std::make_shared(); - for (const auto & partition_id : merge_tree_table->getAllPartitionIds()) - { - auto partition = std::make_shared(); - partition->all = false; - partition->setPartitionID(std::make_shared(partition_id)); - - auto command = std::make_shared(); - command->replace = false; - - command->type = ASTAlterCommand::REPLACE_PARTITION; - command->partition = command->children.emplace_back(std::move(partition)).get(); - - command->from_database = as_database_name; - command->from_table = as_table_saved; - - command->to_database = create.getDatabase(); - command->to_table = create.getTable(); - - command_list->children.push_back(command); - } + command_list->children.push_back(command); auto query = std::make_shared(); query->database = create.database; diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index 58fac744ec0..5424cc1cfc5 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -54,9 +54,7 @@ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_f echo "from foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" echo "from clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree FINAL" -${CLICKHOUSE_CLIENT} -q "SELECT mutation_id, command, block_numbers.partition_id, block_numbers.number, parts_to_do, is_done FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and table = 'clone_as_foo_replacing_merge_tree' ORDER BY mutation_id" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" From 23f0701fc8689d12839c265d894c9aff98f0ae92 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Thu, 5 Sep 2024 08:45:11 +0000 Subject: [PATCH 08/14] Throw an error if CREATE ... CLONE AS ... with a Replicated database --- src/Interpreters/InterpreterCreateQuery.cpp | 7 +++++++ .../03231_create_with_clone_as.reference | 7 +++++++ .../0_stateless/03231_create_with_clone_as.sh | 20 ++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 6506e99a14d..9ba8254630d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1501,6 +1501,13 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } } + if (database && database->getEngineName() == "Replicated" && create.is_clone_as) + { + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); + } + if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { chassert(!ddl_guard); diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index e57d65429cb..a94576269fb 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -22,3 +22,10 @@ from foo_replacing_merge_tree from clone_as_foo_replacing_merge_tree 1 a 2 b +s1 r1 OK 0 0 +CREATE TABLE imdb_03231.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from imdb_03231.foo_merge_tree +1 a +2 b +1 +s1 r1 OK 0 0 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index 5424cc1cfc5..eaa1046b774 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-replicated-database CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -63,4 +64,21 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_file" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" \ No newline at end of file +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" + +# CLONE AS with a Replicated database +${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS imdb_03231" + +${CLICKHOUSE_CLIENT} -q "CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/imdb_03231', 's1', 'r1')" + +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE imdb_03231.foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE imdb_03231.foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO imdb_03231.foo_merge_tree VALUES (1, 'a'), (2, 'b')" +echo "from imdb_03231.foo_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM imdb_03231.foo_merge_tree" +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE imdb_03231.clone_as_foo_merge_tree CLONE AS imdb_03231.foo_merge_tree" 2>&1)" \ + | grep -c 'Code: 344. DB::Exception: .* CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries.' + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS imdb_03231.clone_as_foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS imdb_03231.foo_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS imdb_03231" \ No newline at end of file From 5b4a9bc62bd916051ef4a8711e80eeb4e49ef848 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 6 Sep 2024 06:32:16 +0000 Subject: [PATCH 09/14] Throw an error if CLONE ASfrom a Replicated storage --- src/Interpreters/InterpreterCreateQuery.cpp | 67 +++++++++++-------- .../03231_create_with_clone_as.reference | 4 ++ .../0_stateless/03231_create_with_clone_as.sh | 16 ++++- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9ba8254630d..f29f824eb49 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1468,44 +1468,53 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.select && create.is_materialized_view && mode <= LoadingStrictnessLevel::CREATE) validateMaterializedViewColumnsAndEngine(create, properties, database); - bool allow_heavy_populate = getContext()->getSettingsRef().database_replicated_allow_heavy_create && create.is_populate; - if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + bool is_storage_replicated = false; + + if (create.storage && isReplicated(*create.storage)) + is_storage_replicated = true; + + if (create.targets) { - bool is_storage_replicated = false; - - if (create.storage && isReplicated(*create.storage)) - is_storage_replicated = true; - - if (create.targets) + for (const auto & inner_table_engine : create.targets->getInnerEngines()) { - for (const auto & inner_table_engine : create.targets->getInnerEngines()) + if (isReplicated(*inner_table_engine)) + is_storage_replicated = true; + } + } + + bool allow_heavy_populate = getContext()->getSettingsRef().database_replicated_allow_heavy_create && create.is_populate; + if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + { + const bool allow_create_select_for_replicated + = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; + if (!allow_create_select_for_replicated) { - if (isReplicated(*inner_table_engine)) - is_storage_replicated = true; + /// POPULATE can be enabled with setting, provide hint in error message + if (create.is_populate) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " + "queries. " + "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " + "caution"); + + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); } } - const bool allow_create_select_for_replicated = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; - if (!allow_create_select_for_replicated) - { - /// POPULATE can be enabled with setting, provide hint in error message - if (create.is_populate) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT queries. " - "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with caution"); - + if (create.is_clone_as) + { + if (database && database->getEngineName() == "Replicated") throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - } - } + "CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - if (database && database->getEngineName() == "Replicated" && create.is_clone_as) - { - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); + if (is_storage_replicated) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE CLONE AS is not supported with Replicated storages. Consider using separate CREATE and INSERT queries."); } if (database && database->shouldReplicateQuery(getContext(), query_ptr)) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index a94576269fb..4aed6849728 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -22,6 +22,10 @@ from foo_replacing_merge_tree from clone_as_foo_replacing_merge_tree 1 a 2 b +CREATE TABLE default.foo_replicated_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/test_foo_replicated_merge_tree\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +1 s1 r1 OK 0 0 CREATE TABLE imdb_03231.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 from imdb_03231.foo_merge_tree diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh index eaa1046b774..8e2cf30abce 100755 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sh @@ -13,6 +13,8 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replicated_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree" # CLONE AS with a table of Memory engine ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory" @@ -55,7 +57,15 @@ ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_f echo "from foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" echo "from clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree FINAL" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" + +# CLONE AS with a table of ReplicatedMergeTree engine +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_replicated_merge_tree" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replicated_merge_tree VALUES (1, 'a'), (2, 'b')" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replicated_merge_tree" +echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree" 2>&1)" \ + | grep -c 'Code: 344. DB::Exception: .* CREATE CLONE AS is not supported with Replicated storages. Consider using separate CREATE and INSERT queries.' ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" @@ -65,11 +75,13 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replicated_merge_tree" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree" # CLONE AS with a Replicated database ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS imdb_03231" -${CLICKHOUSE_CLIENT} -q "CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/imdb_03231', 's1', 'r1')" +${CLICKHOUSE_CLIENT} -q "CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/$CLICKHOUSE_DATABASE/imdb_03231', 's1', 'r1')" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE imdb_03231.foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x" ${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE imdb_03231.foo_merge_tree" From 4aa9459c63dfdc2f0b2835bba4dc3c6ed905d75b Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 6 Sep 2024 06:39:00 +0000 Subject: [PATCH 10/14] Change the test file from .sh to .sql --- .../03231_create_with_clone_as.reference | 4 - .../0_stateless/03231_create_with_clone_as.sh | 96 ------------------- .../03231_create_with_clone_as.sql | 87 +++++++++++++++++ 3 files changed, 87 insertions(+), 100 deletions(-) delete mode 100755 tests/queries/0_stateless/03231_create_with_clone_as.sh create mode 100644 tests/queries/0_stateless/03231_create_with_clone_as.sql diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index 4aed6849728..34f39fe932a 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -1,7 +1,5 @@ CREATE TABLE default.foo_memory\n(\n `x` Int8,\n `y` String\n)\nENGINE = Memory -1 CREATE TABLE default.foo_file\n(\n `x` Int8,\n `y` String\n)\nENGINE = File(\'TabSeparated\') -1 CREATE TABLE default.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 a 2 b @@ -25,11 +23,9 @@ from clone_as_foo_replacing_merge_tree CREATE TABLE default.foo_replicated_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/test_foo_replicated_merge_tree\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 a 2 b -1 s1 r1 OK 0 0 CREATE TABLE imdb_03231.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 from imdb_03231.foo_merge_tree 1 a 2 b -1 s1 r1 OK 0 0 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sh b/tests/queries/0_stateless/03231_create_with_clone_as.sh deleted file mode 100755 index 8e2cf30abce..00000000000 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-replicated-database - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_file" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_file" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replicated_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree" - -# CLONE AS with a table of Memory engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_memory" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_memory VALUES (1, 'a'), (2, 'b')" - -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_memory CLONE AS foo_memory" 2>&1)" \ - | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' - - -# CLONE AS with a table of File engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_file (x Int8, y String) ENGINE=File(TabSeparated)" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_file" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_file VALUES (1, 'a'), (2, 'b')" - -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_file CLONE AS foo_file" 2>&1)" \ - | grep -c 'Code: 344. DB::Exception: .* Only support CLONE AS with tables of the MergeTree family' - -# CLONE AS with a table of MergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b')" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" - -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_merge_tree" -echo "from foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_merge_tree" -echo "from clone_as_foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_merge_tree" - -# CLONE AS with a table of ReplacingMergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replacing_merge_tree VALUES (1, 'a'), (2, 'b')" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" - -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE clone_as_foo_replacing_merge_tree" -echo "from foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replacing_merge_tree" -echo "from clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM clone_as_foo_replacing_merge_tree" - -# CLONE AS with a table of ReplicatedMergeTree engine -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE foo_replicated_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO foo_replicated_merge_tree VALUES (1, 'a'), (2, 'b')" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM foo_replicated_merge_tree" -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree" 2>&1)" \ - | grep -c 'Code: 344. DB::Exception: .* CREATE CLONE AS is not supported with Replicated storages. Consider using separate CREATE and INSERT queries.' - -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_memory" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_memory" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_file" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_file" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS foo_replicated_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree" - -# CLONE AS with a Replicated database -${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS imdb_03231" - -${CLICKHOUSE_CLIENT} -q "CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/$CLICKHOUSE_DATABASE/imdb_03231', 's1', 'r1')" - -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "CREATE TABLE imdb_03231.foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SHOW CREATE TABLE imdb_03231.foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "INSERT INTO imdb_03231.foo_merge_tree VALUES (1, 'a'), (2, 'b')" -echo "from imdb_03231.foo_merge_tree" -${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 -q "SELECT * FROM imdb_03231.foo_merge_tree" -echo "$(${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null -q "CREATE TABLE imdb_03231.clone_as_foo_merge_tree CLONE AS imdb_03231.foo_merge_tree" 2>&1)" \ - | grep -c 'Code: 344. DB::Exception: .* CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries.' - -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS imdb_03231.clone_as_foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS imdb_03231.foo_merge_tree" -${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS imdb_03231" \ No newline at end of file diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sql b/tests/queries/0_stateless/03231_create_with_clone_as.sql new file mode 100644 index 00000000000..b410c09bb9c --- /dev/null +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sql @@ -0,0 +1,87 @@ +-- Tags: no-replicated-database +-- Tag no-replicated-database: Unsupported type of CREATE TABLE ... CLONE AS ... query + +DROP TABLE IF EXISTS foo_memory; +DROP TABLE IF EXISTS clone_as_foo_memory; +DROP TABLE IF EXISTS foo_file; +DROP TABLE IF EXISTS clone_as_foo_file; +DROP TABLE IF EXISTS foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; + +-- CLONE AS with a table of Memory engine +CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory; +SHOW CREATE TABLE foo_memory; +INSERT INTO foo_memory VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE clone_as_foo_memory CLONE AS foo_memory; -- { serverError SUPPORT_IS_DISABLED } + +-- CLONE AS with a table of File engine +CREATE TABLE foo_file (x Int8, y String) ENGINE=File(TabSeparated); +SHOW CREATE TABLE foo_file; +INSERT INTO foo_file VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE clone_as_foo_file CLONE AS foo_file; -- { serverError SUPPORT_IS_DISABLED } + +-- CLONE AS with a table of MergeTree engine +CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_merge_tree; +INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT * FROM foo_merge_tree; + +CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree; +SHOW CREATE TABLE clone_as_foo_merge_tree; +SELECT 'from foo_merge_tree'; +SELECT * FROM foo_merge_tree; +SELECT 'from clone_as_foo_merge_tree'; +SELECT * FROM clone_as_foo_merge_tree; + +-- CLONE AS with a table of ReplacingMergeTree engine +CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_replacing_merge_tree; +INSERT INTO foo_replacing_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT * FROM foo_replacing_merge_tree; + +CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree; +SHOW CREATE TABLE clone_as_foo_replacing_merge_tree; +SELECT 'from foo_replacing_merge_tree'; +SELECT * FROM foo_replacing_merge_tree; +SELECT 'from clone_as_foo_replacing_merge_tree'; +SELECT * FROM clone_as_foo_replacing_merge_tree; + +-- CLONE AS with a table of ReplicatedMergeTree engine +CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x; +SHOW CREATE TABLE foo_replicated_merge_tree; +INSERT INTO foo_replicated_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT * FROM foo_replicated_merge_tree; +CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree; -- { serverError SUPPORT_IS_DISABLED } + +DROP TABLE IF EXISTS foo_memory; +DROP TABLE IF EXISTS clone_as_foo_memory; +DROP TABLE IF EXISTS foo_file; +DROP TABLE IF EXISTS clone_as_foo_file; +DROP TABLE IF EXISTS foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; + +-- CLONE AS with a Replicated database +DROP DATABASE IF EXISTS imdb_03231; + +CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/{database}/imdb_03231', 's1', 'r1'); + +CREATE TABLE imdb_03231.foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; +SHOW CREATE TABLE imdb_03231.foo_merge_tree; +INSERT INTO imdb_03231.foo_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT 'from imdb_03231.foo_merge_tree'; +SELECT * FROM imdb_03231.foo_merge_tree; +CREATE TABLE imdb_03231.clone_as_foo_merge_tree CLONE AS imdb_03231.foo_merge_tree; -- { serverError SUPPORT_IS_DISABLED } + +DROP TABLE IF EXISTS imdb_03231.clone_as_foo_merge_tree; +DROP TABLE IF EXISTS imdb_03231.foo_merge_tree; +DROP DATABASE IF EXISTS imdb_03231; \ No newline at end of file From c01e63fd2b788c3554c4e5e883692513dda2e91d Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Fri, 6 Sep 2024 15:10:35 +0000 Subject: [PATCH 11/14] 1) Update checking MergeTree storage. 2) Remove unused code segment. 3) Update the test --- src/Interpreters/InterpreterCreateQuery.cpp | 21 ++++------------- .../03231_create_with_clone_as.reference | 4 ++-- .../03231_create_with_clone_as.sql | 23 ++++++++++--------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f29f824eb49..007fec6f087 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -811,6 +811,11 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + if (create.is_clone_as && !endsWith(as_storage->getName(), "MergeTree")) + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); + } + auto as_storage_metadata = as_storage->getInMemoryMetadataPtr(); properties.columns = as_storage_metadata->getColumns(); @@ -828,10 +833,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti { /// Only MergeTree support TTL properties.columns.resetColumnTTLs(); - if (create.is_clone_as) - { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); - } } properties.constraints = as_storage_metadata->getConstraints(); @@ -1960,18 +1961,6 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) && (!(create.is_materialized_view || create.is_window_view) || create.is_populate)) { String as_database_name = getContext()->resolveDatabase(create.as_database); - StorageID as_table_id = {as_database_name, as_table_saved}; - StoragePtr as_table = DatabaseCatalog::instance().tryGetTable(as_table_id, getContext()); - if (!as_table) - { - return {}; - } - - auto merge_tree_table = std::dynamic_pointer_cast(as_table); - if (!merge_tree_table) - { - return {}; - } auto partition = std::make_shared(); partition->all = true; diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index 34f39fe932a..e0d292ec5d4 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -24,8 +24,8 @@ CREATE TABLE default.foo_replicated_merge_tree\n(\n `x` Int8,\n `y` String 1 a 2 b s1 r1 OK 0 0 -CREATE TABLE imdb_03231.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 -from imdb_03231.foo_merge_tree +CREATE TABLE default_1.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_merge_tree 1 a 2 b s1 r1 OK 0 0 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sql b/tests/queries/0_stateless/03231_create_with_clone_as.sql index b410c09bb9c..712e90b2f8a 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sql +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sql @@ -71,17 +71,18 @@ DROP TABLE IF EXISTS foo_replicated_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; -- CLONE AS with a Replicated database -DROP DATABASE IF EXISTS imdb_03231; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; -CREATE DATABASE imdb_03231 ENGINE = Replicated('/test/databases/{database}/imdb_03231', 's1', 'r1'); +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Replicated('/test/databases/{database}/test_03231', 's1', 'r1'); +USE {CLICKHOUSE_DATABASE_1:Identifier}; -CREATE TABLE imdb_03231.foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; -SHOW CREATE TABLE imdb_03231.foo_merge_tree; -INSERT INTO imdb_03231.foo_merge_tree VALUES (1, 'a'), (2, 'b'); -SELECT 'from imdb_03231.foo_merge_tree'; -SELECT * FROM imdb_03231.foo_merge_tree; -CREATE TABLE imdb_03231.clone_as_foo_merge_tree CLONE AS imdb_03231.foo_merge_tree; -- { serverError SUPPORT_IS_DISABLED } +CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_merge_tree; +INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT 'from foo_merge_tree'; +SELECT * FROM foo_merge_tree; +CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree; -- { serverError SUPPORT_IS_DISABLED } -DROP TABLE IF EXISTS imdb_03231.clone_as_foo_merge_tree; -DROP TABLE IF EXISTS imdb_03231.foo_merge_tree; -DROP DATABASE IF EXISTS imdb_03231; \ No newline at end of file +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS foo_merge_tree; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; \ No newline at end of file From 841dc275bd4588a290fd1763365ba1eb94b1b2da Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Mon, 9 Sep 2024 08:05:29 +0000 Subject: [PATCH 12/14] 1) Throw an error when creating a table with CLONE AS and specifying ENGINE. 2) Add document for creating a table with Schema and Data CLoned from another table. --- docs/en/sql-reference/statements/create/table.md | 13 +++++++++++++ src/Interpreters/InterpreterCreateQuery.cpp | 4 ++++ .../0_stateless/03231_create_with_clone_as.sql | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 7428e6cd6ca..74e422faae4 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -43,6 +43,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. +### With a Schema and Data Cloned from Another Table + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 +``` + +Creates a table with the same structure as another table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following: + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2; +ALTER TABLE [db.]table_name ATTACH PARTITION ALL FROM [db2].name2; +``` + ### From a Table Function ``` sql diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 007fec6f087..671226e3ea7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1139,6 +1139,10 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.storage) { + /// When creating a table with CLONE AS, the structure of the new table must be copied from the source table's structure. So that the data could be cloned later. + if (create.is_clone_as) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot specify ENGINE when creating a table with CLONE AS"); + /// This table already has a storage definition. if (!create.storage->engine) { diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sql b/tests/queries/0_stateless/03231_create_with_clone_as.sql index 712e90b2f8a..31dc5c22c86 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sql +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sql @@ -7,8 +7,12 @@ DROP TABLE IF EXISTS foo_file; DROP TABLE IF EXISTS clone_as_foo_file; DROP TABLE IF EXISTS foo_merge_tree; DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_y; DROP TABLE IF EXISTS foo_replacing_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_y; DROP TABLE IF EXISTS foo_replicated_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; @@ -39,6 +43,10 @@ SELECT * FROM foo_merge_tree; SELECT 'from clone_as_foo_merge_tree'; SELECT * FROM clone_as_foo_merge_tree; +-- Specify ENGINE +CREATE TABLE clone_as_foo_merge_tree_p_x CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY x; -- { serverError INCORRECT_QUERY } +CREATE TABLE clone_as_foo_merge_tree_p_y CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY y; -- { serverError INCORRECT_QUERY } + -- CLONE AS with a table of ReplacingMergeTree engine CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x; SHOW CREATE TABLE foo_replacing_merge_tree; @@ -52,6 +60,10 @@ SELECT * FROM foo_replacing_merge_tree; SELECT 'from clone_as_foo_replacing_merge_tree'; SELECT * FROM clone_as_foo_replacing_merge_tree; +-- Specify ENGINE +CREATE TABLE clone_as_foo_replacing_merge_tree_x CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY x; -- { serverError INCORRECT_QUERY } +CREATE TABLE clone_as_foo_replacing_merge_tree_y CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY y; -- { serverError INCORRECT_QUERY } + -- CLONE AS with a table of ReplicatedMergeTree engine CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x; SHOW CREATE TABLE foo_replicated_merge_tree; @@ -65,8 +77,12 @@ DROP TABLE IF EXISTS foo_file; DROP TABLE IF EXISTS clone_as_foo_file; DROP TABLE IF EXISTS foo_merge_tree; DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_y; DROP TABLE IF EXISTS foo_replacing_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_y; DROP TABLE IF EXISTS foo_replicated_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; From 0b7f28f75b940c104efaf1ba7e903aef2a7e9339 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Tue, 10 Sep 2024 07:55:21 +0000 Subject: [PATCH 13/14] 1) Support CLONE AS with ReplicatedMergeTree 2) Support CLONE AS with specifying ENGINE --- src/Interpreters/InterpreterCreateQuery.cpp | 52 ++++++++++++++----- .../03231_create_with_clone_as.reference | 11 ++++ .../03231_create_with_clone_as.sql | 43 ++++++++++----- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 671226e3ea7..80441897ed4 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -811,10 +812,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - if (create.is_clone_as && !endsWith(as_storage->getName(), "MergeTree")) - { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); - } + auto as_storage_metadata = as_storage->getInMemoryMetadataPtr(); properties.columns = as_storage_metadata->getColumns(); @@ -836,6 +834,43 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti } properties.constraints = as_storage_metadata->getConstraints(); + + if (create.is_clone_as) + { + if (!endsWith(as_storage->getName(), "MergeTree")) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS from tables of the MergeTree family"); + + if (create.storage) + { + if (!endsWith(create.storage->engine->name, "MergeTree")) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); + + /// Ensure that as_storage and the new storage has the same primary key, sorting key and partition key + auto query_to_string = [](const IAST * ast) { return ast ? queryToString(*ast) : ""; }; + + const String as_storage_sorting_key_str = query_to_string(as_storage_metadata->getSortingKeyAST().get()); + const String as_storage_primary_key_str = query_to_string(as_storage_metadata->getPrimaryKeyAST().get()); + const String as_storage_partition_key_str = query_to_string(as_storage_metadata->getPartitionKeyAST().get()); + + const String storage_sorting_key_str = query_to_string(create.storage->order_by); + const String storage_primary_key_str = query_to_string(create.storage->primary_key); + const String storage_partition_key_str = query_to_string(create.storage->partition_by); + + if (as_storage_sorting_key_str != storage_sorting_key_str) + { + /// It is possible that the storage only has primary key and an empty sorting key, and as_storage has both primary key and sorting key with the same value. + if (as_storage_sorting_key_str != as_storage_primary_key_str || as_storage_sorting_key_str != storage_primary_key_str) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different ordering"); + } + } + if (as_storage_partition_key_str != storage_partition_key_str) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + + if (as_storage_primary_key_str != storage_primary_key_str) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different primary key"); + } + } } else if (create.select) { @@ -1139,10 +1174,6 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.storage) { - /// When creating a table with CLONE AS, the structure of the new table must be copied from the source table's structure. So that the data could be cloned later. - if (create.is_clone_as) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot specify ENGINE when creating a table with CLONE AS"); - /// This table already has a storage definition. if (!create.storage->engine) { @@ -1515,11 +1546,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, "CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - - if (is_storage_replicated) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE CLONE AS is not supported with Replicated storages. Consider using separate CREATE and INSERT queries."); } if (database && database->shouldReplicateQuery(getContext(), query_ptr)) diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference index e0d292ec5d4..d2046e19a23 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.reference +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -10,6 +10,9 @@ from foo_merge_tree from clone_as_foo_merge_tree 1 a 2 b +from clone_as_foo_merge_tree_p_x +1 a +2 b CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 1 a 2 b @@ -20,7 +23,15 @@ from foo_replacing_merge_tree from clone_as_foo_replacing_merge_tree 1 a 2 b +from clone_as_foo_replacing_merge_tree_p_x +1 a +2 b CREATE TABLE default.foo_replicated_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/test_foo_replicated_merge_tree\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_replicated_merge_tree +1 a +2 b +CREATE TABLE default.clone_as_foo_replicated_merge_tree_p_x\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/clone_as_foo_replicated_merge_tree_p_x\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from clone_as_foo_replicated_merge_tree_p_x 1 a 2 b s1 r1 OK 0 0 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sql b/tests/queries/0_stateless/03231_create_with_clone_as.sql index 31dc5c22c86..c5793206e88 100644 --- a/tests/queries/0_stateless/03231_create_with_clone_as.sql +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sql @@ -7,14 +7,16 @@ DROP TABLE IF EXISTS foo_file; DROP TABLE IF EXISTS clone_as_foo_file; DROP TABLE IF EXISTS foo_merge_tree; DROP TABLE IF EXISTS clone_as_foo_merge_tree; -DROP TABLE IF EXISTS clone_as_foo_merge_tree_x; -DROP TABLE IF EXISTS clone_as_foo_merge_tree_y; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_y; DROP TABLE IF EXISTS foo_replacing_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; -DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_x; -DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_y; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_y; DROP TABLE IF EXISTS foo_replicated_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_y; -- CLONE AS with a table of Memory engine CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory; @@ -44,8 +46,10 @@ SELECT 'from clone_as_foo_merge_tree'; SELECT * FROM clone_as_foo_merge_tree; -- Specify ENGINE -CREATE TABLE clone_as_foo_merge_tree_p_x CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY x; -- { serverError INCORRECT_QUERY } -CREATE TABLE clone_as_foo_merge_tree_p_y CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY y; -- { serverError INCORRECT_QUERY } +CREATE TABLE clone_as_foo_merge_tree_p_x CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY x; +SELECT 'from clone_as_foo_merge_tree_p_x'; +SELECT * FROM clone_as_foo_merge_tree_p_x; +CREATE TABLE clone_as_foo_merge_tree_p_y CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } -- CLONE AS with a table of ReplacingMergeTree engine CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x; @@ -61,15 +65,26 @@ SELECT 'from clone_as_foo_replacing_merge_tree'; SELECT * FROM clone_as_foo_replacing_merge_tree; -- Specify ENGINE -CREATE TABLE clone_as_foo_replacing_merge_tree_x CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY x; -- { serverError INCORRECT_QUERY } -CREATE TABLE clone_as_foo_replacing_merge_tree_y CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY y; -- { serverError INCORRECT_QUERY } +CREATE TABLE clone_as_foo_replacing_merge_tree_p_x CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY x; +SELECT 'from clone_as_foo_replacing_merge_tree_p_x'; +SELECT * FROM clone_as_foo_replacing_merge_tree_p_x; +CREATE TABLE clone_as_foo_replacing_merge_tree_p_y CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } -- CLONE AS with a table of ReplicatedMergeTree engine CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x; SHOW CREATE TABLE foo_replicated_merge_tree; INSERT INTO foo_replicated_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT 'from foo_replicated_merge_tree'; SELECT * FROM foo_replicated_merge_tree; -CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree; -- { serverError SUPPORT_IS_DISABLED } + +CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree; -- { serverError REPLICA_ALREADY_EXISTS } + +-- Specify ENGINE +CREATE TABLE clone_as_foo_replicated_merge_tree_p_x CLONE AS foo_replicated_merge_tree ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/clone_as_foo_replicated_merge_tree_p_x', 'r1') PRIMARY KEY x; +SHOW CREATE TABLE clone_as_foo_replicated_merge_tree_p_x; +SELECT 'from clone_as_foo_replicated_merge_tree_p_x'; +SELECT * FROM foo_replicated_merge_tree; +CREATE TABLE clone_as_foo_replicated_merge_tree_p_y CLONE AS foo_replicated_merge_tree ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/clone_as_foo_replicated_merge_tree_p_y', 'r1') PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } DROP TABLE IF EXISTS foo_memory; DROP TABLE IF EXISTS clone_as_foo_memory; @@ -77,14 +92,16 @@ DROP TABLE IF EXISTS foo_file; DROP TABLE IF EXISTS clone_as_foo_file; DROP TABLE IF EXISTS foo_merge_tree; DROP TABLE IF EXISTS clone_as_foo_merge_tree; -DROP TABLE IF EXISTS clone_as_foo_merge_tree_x; -DROP TABLE IF EXISTS clone_as_foo_merge_tree_y; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_y; DROP TABLE IF EXISTS foo_replacing_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; -DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_x; -DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_y; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_y; DROP TABLE IF EXISTS foo_replicated_merge_tree; DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_y; -- CLONE AS with a Replicated database DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; From 561daee66f170abf5334987afd2cf62ab93840f5 Mon Sep 17 00:00:00 2001 From: Tuan Pham Anh Date: Tue, 10 Sep 2024 10:24:31 +0000 Subject: [PATCH 14/14] Update document --- docs/en/sql-reference/statements/create/table.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 74e422faae4..250cddb09dc 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -46,13 +46,13 @@ Creates a table with the same structure as another table. You can specify a diff ### With a Schema and Data Cloned from Another Table ``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 +CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 [ENGINE = engine] ``` -Creates a table with the same structure as another table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following: +Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following: ``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2; +CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine]; ALTER TABLE [db.]table_name ATTACH PARTITION ALL FROM [db2].name2; ```