From 0b70bffe36e8b7d57a917f3d921c2cd527d2a7be Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Dec 2019 22:48:16 +0300 Subject: [PATCH] use StorageID in views --- .../PushingToViewsBlockOutputStream.cpp | 4 +- dbms/src/Databases/DatabaseOnDisk.cpp | 1 - dbms/src/Interpreters/Context.cpp | 34 ++-- dbms/src/Interpreters/Context.h | 9 +- .../Interpreters/InterpreterCreateQuery.cpp | 8 +- dbms/src/Parsers/ASTAlterQuery.h | 1 + dbms/src/Parsers/ASTCreateQuery.cpp | 13 +- dbms/src/Parsers/ASTCreateQuery.h | 5 +- dbms/src/Parsers/ASTQueryWithTableAndOutput.h | 1 + dbms/src/Parsers/ExpressionElementParsers.cpp | 36 +++- dbms/src/Parsers/ExpressionElementParsers.h | 6 + dbms/src/Parsers/ParserCreateQuery.cpp | 125 +++---------- dbms/src/Storages/Kafka/StorageKafka.cpp | 4 +- .../src/Storages/LiveView/StorageLiveView.cpp | 70 ++------ dbms/src/Storages/LiveView/StorageLiveView.h | 6 +- dbms/src/Storages/StorageID.h | 38 +++- dbms/src/Storages/StorageMaterializedView.cpp | 170 +++++++----------- dbms/src/Storages/StorageMaterializedView.h | 9 +- 18 files changed, 229 insertions(+), 311 deletions(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index c43f2e7bf1d..1e197d4d182 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -46,7 +46,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( for (const auto & database_table : dependencies) { - auto dependent_table = context.getTable(database_table.database_name, database_table.table_name); //FIXME + auto dependent_table = context.getTable(database_table); ASTPtr query; BlockOutputStreamPtr out; @@ -219,7 +219,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n /// InterpreterSelectQuery will do processing of alias columns. Context local_context = *views_context; local_context.addViewSource( - StorageValues::create(StorageID(storage->getDatabaseName(), storage->getTableName()), storage->getColumns(), //FIXME + StorageValues::create(storage->getStorageID(), storage->getColumns(), block)); select.emplace(view.query, local_context, SelectQueryOptions()); in = std::make_shared(select->execute().in); diff --git a/dbms/src/Databases/DatabaseOnDisk.cpp b/dbms/src/Databases/DatabaseOnDisk.cpp index 700a7ea7b71..44c6dddb886 100644 --- a/dbms/src/Databases/DatabaseOnDisk.cpp +++ b/dbms/src/Databases/DatabaseOnDisk.cpp @@ -29,7 +29,6 @@ namespace DB { static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; -static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; namespace ErrorCodes { diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 266e932ca17..227a6a6fa1d 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -728,7 +728,6 @@ void Context::checkDatabaseAccessRightsImpl(const std::string & database_name) c throw Exception("Access denied to database " + database_name + " for user " + client_info.current_user , ErrorCodes::DATABASE_ACCESS_DENIED); } -//FIXME use uuids if not empty void Context::addDependencyUnsafe(const StorageID & from, const StorageID & where) { @@ -737,7 +736,7 @@ void Context::addDependencyUnsafe(const StorageID & from, const StorageID & wher shared->view_dependencies[from].insert(where); // Notify table of dependencies change - auto table = tryGetTable(from.database_name, from.table_name); + auto table = tryGetTable(from); if (table != nullptr) table->updateDependencies(); } @@ -755,7 +754,7 @@ void Context::removeDependencyUnsafe(const StorageID & from, const StorageID & w shared->view_dependencies[from].erase(where); // Notify table of dependencies change - auto table = tryGetTable(from.database_name, from.table_name); + auto table = tryGetTable(from); if (table != nullptr) table->updateDependencies(); } @@ -910,24 +909,32 @@ StoragePtr Context::tryGetExternalTable(const String & table_name) const return jt->second.first; } - StoragePtr Context::getTable(const String & database_name, const String & table_name) const +{ + return getTable(StorageID(database_name, table_name)); +} + +StoragePtr Context::getTable(const StorageID & table_id) const { Exception exc; - auto res = getTableImpl(database_name, table_name, &exc); + auto res = getTableImpl(table_id, &exc); if (!res) throw exc; return res; } - StoragePtr Context::tryGetTable(const String & database_name, const String & table_name) const { - return getTableImpl(database_name, table_name, nullptr); + return tryGetTable(StorageID(database_name, table_name)); +} + +StoragePtr Context::tryGetTable(const StorageID & table_id) const +{ + return getTableImpl(table_id, nullptr); } -StoragePtr Context::getTableImpl(const String & database_name, const String & table_name, Exception * exception) const +StoragePtr Context::getTableImpl(const StorageID & table_id, Exception * exception) const { String db; DatabasePtr database; @@ -935,14 +942,15 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta { auto lock = getLock(); - if (database_name.empty()) + if (table_id.database_name.empty()) { - StoragePtr res = tryGetExternalTable(table_name); + StoragePtr res = tryGetExternalTable(table_id.table_name); if (res) return res; } - db = resolveDatabase(database_name, current_database); + //FIXME what if table was moved to another database? + db = resolveDatabase(table_id.database_name, current_database); checkDatabaseAccessRightsImpl(db); Databases::const_iterator it = shared->databases.find(db); @@ -956,11 +964,11 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta database = it->second; } - auto table = database->tryGetTable(*this, table_name); + auto table = database->tryGetTable(*this, table_id.table_name); if (!table) { if (exception) - *exception = Exception("Table " + backQuoteIfNeed(db) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + *exception = Exception("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); return {}; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index e77b8995439..7571843abfd 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -88,10 +88,6 @@ class CompiledExpressionCache; #endif -/// (database name, table name) -//FIXME replace with StorageID -//using DatabaseAndTableName = std::pair; - /// Table -> set of table-views that make SELECT from it. using ViewDependencies = std::map>; using Dependencies = std::vector; @@ -125,6 +121,7 @@ using IHostContextPtr = std::shared_ptr; * * Everything is encapsulated for all sorts of checks and locks. */ +///TODO remove syntax sugar and legacy methods from Context (e.g. getInputFormat(...) which just returns object from factory) class Context { private: @@ -276,7 +273,9 @@ public: Tables getExternalTables() const; StoragePtr tryGetExternalTable(const String & table_name) const; StoragePtr getTable(const String & database_name, const String & table_name) const; + StoragePtr getTable(const StorageID & table_id) const; StoragePtr tryGetTable(const String & database_name, const String & table_name) const; + StoragePtr tryGetTable(const StorageID & table_id) const; void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; @@ -578,7 +577,7 @@ private: EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const; - StoragePtr getTableImpl(const String & database_name, const String & table_name, Exception * exception) const; + StoragePtr getTableImpl(const StorageID & table_id, Exception * exception) const; SessionKey getSessionKey(const String & session_id) const; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index d58aa1530d3..658f7ff80c1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -548,8 +548,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!create.cluster.empty()) { NameSet databases{create.database}; - if (!create.to_table.empty()) - databases.emplace(create.to_database); + if (!create.to_table_id.empty()) + databases.emplace(create.to_table_id.database_name); /// NOTE: if it's CREATE query and create.database is DatabaseAtomic, different UUIDs will be generated on all servers. /// However, it allows to use UUID as replica name. @@ -577,8 +577,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) String current_database = context.getCurrentDatabase(); if (create.database.empty()) create.database = current_database; - if (create.to_database.empty()) - create.to_database = current_database; + if (!create.to_table_id.empty() && create.to_table_id.database_name.empty()) + create.to_table_id.database_name = current_database; if (create.select && (create.is_view || create.is_materialized_view || create.is_live_view)) { diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 162b9518824..e2b8eb20e35 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -147,6 +147,7 @@ public: String with_name; /// REPLACE(ATTACH) PARTITION partition FROM db.table + //FIXME use StorageID String from_database; String from_table; /// To distinguish REPLACE and ATTACH PARTITION partition FROM db.table diff --git a/dbms/src/Parsers/ASTCreateQuery.cpp b/dbms/src/Parsers/ASTCreateQuery.cpp index 32ac43129f8..62a44914f12 100644 --- a/dbms/src/Parsers/ASTCreateQuery.cpp +++ b/dbms/src/Parsers/ASTCreateQuery.cpp @@ -233,9 +233,11 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat << what << " " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : "") - << (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table) - << (!uuid.empty() ? " UUID " + quoteString(uuid) : ""); - formatOnCluster(settings); + << (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table); + if (!uuid.empty()) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "") + << quoteString(uuid); + formatOnCluster(settings); } else { @@ -250,11 +252,12 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); as_table_function->formatImpl(settings, state, frame); } - if (!to_table.empty()) + if (!to_table_id.empty()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : "") - << (!to_database.empty() ? backQuoteIfNeed(to_database) + "." : "") << backQuoteIfNeed(to_table); + << (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "") + << backQuoteIfNeed(to_table_id.table_name); } if (!as_table.empty()) diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 9e88e51bdd6..87097e36882 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -65,15 +66,13 @@ public: ASTColumns * columns_list = nullptr; ASTExpressionList * dictionary_attributes_list = nullptr; /// attributes of dictionary ASTExpressionList * tables = nullptr; - String to_database; /// For CREATE MATERIALIZED VIEW mv TO table. - String to_table; + StorageID to_table_id; /// For CREATE MATERIALIZED VIEW mv TO table. ASTStorage * storage = nullptr; String as_database; String as_table; ASTPtr as_table_function; ASTSelectWithUnionQuery * select = nullptr; ASTDictionary * dictionary = nullptr; /// dictionary definition (layout, primary key, etc.) - String uuid; /// For ATTACH TABLE query when db engine is Atomic /** Get the text that identifies this element. */ String getID(char delim) const override { return (attach ? "AttachQuery" : "CreateQuery") + (delim + database) + delim + table; } diff --git a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h index 594876ace7b..5068db1edb7 100644 --- a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h +++ b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h @@ -15,6 +15,7 @@ class ASTQueryWithTableAndOutput : public ASTQueryWithOutput public: String database; String table; + String uuid; bool temporary{false}; protected: diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index c26f9363797..5c6f8ca8593 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -29,7 +29,7 @@ #include #include -#include +#include #include "ASTColumnsMatcher.h" @@ -197,6 +197,40 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex } +bool parseStorageID(IParser::Pos & pos, StorageID & res, Expected & expected) +{ + ParserKeyword s_uuid("UUID"); + ParserIdentifier name_p; + ParserStringLiteral uuid_p; + ParserToken s_dot(TokenType::Dot); + + ASTPtr database; + ASTPtr table; + ASTPtr uuid; + + if (!name_p.parse(pos, table, expected)) + return false; + + if (s_dot.ignore(pos, expected)) + { + database = table; + if (!name_p.parse(pos, table, expected)) + return false; + } + + if (s_uuid.ignore(pos, expected)) + { + if (!uuid_p.parse(pos, uuid, expected)) + return false; + } + + tryGetIdentifierNameInto(database, res.database_name); + tryGetIdentifierNameInto(table, res.table_name); + res.uuid = uuid ? uuid->as()->value.get() : ""; + return true; +} + + bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserIdentifier id_parser; diff --git a/dbms/src/Parsers/ExpressionElementParsers.h b/dbms/src/Parsers/ExpressionElementParsers.h index 63ed1348b13..35e8948938d 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.h +++ b/dbms/src/Parsers/ExpressionElementParsers.h @@ -56,6 +56,12 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; +struct StorageID; +/// Table name, possibly with database name and UUID as string literal +/// [db_name.]table_name [UUID 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'] +//TODO replace with class +bool parseStorageID(IParser::Pos & pos, StorageID & res, Expected & expected); + /// Just * class ParserAsterisk : public IParserBase { diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index aa0af4ea739..6a61728a7a6 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace DB @@ -340,7 +341,6 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ParserKeyword s_temporary("TEMPORARY"); ParserKeyword s_table("TABLE"); ParserKeyword s_if_not_exists("IF NOT EXISTS"); - ParserKeyword s_uuid("UUID"); ParserKeyword s_on("ON"); ParserKeyword s_as("AS"); ParserToken s_dot(TokenType::Dot); @@ -352,14 +352,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ParserSelectWithUnionQuery select_p; ParserFunction table_function_p; ParserNameList names_p; - ParserStringLiteral uuid_p; - ASTPtr database; - ASTPtr table; - ASTPtr uuid; + StorageID table_id; ASTPtr columns_list; - ASTPtr to_database; - ASTPtr to_table; ASTPtr storage; ASTPtr as_database; ASTPtr as_table; @@ -389,22 +384,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (s_if_not_exists.ignore(pos, expected)) if_not_exists = true; - if (!name_p.parse(pos, table, expected)) + if (!parseStorageID(pos, table_id, expected)) return false; - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } - - if (attach && s_uuid.ignore(pos, expected)) - { - if (!uuid_p.parse(pos, uuid, expected)) - return false; - } - if (s_on.ignore(pos, expected)) { if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) @@ -421,8 +403,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe query->if_not_exists = if_not_exists; query->cluster = cluster_str; - tryGetIdentifierNameInto(database, query->database); - tryGetIdentifierNameInto(table, query->table); + query->database = table_id.database_name; + query->table = table_id.table_name; + query->uuid = table_id.uuid; return true; } @@ -479,15 +462,11 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe query->if_not_exists = if_not_exists; query->temporary = is_temporary; - tryGetIdentifierNameInto(database, query->database); - tryGetIdentifierNameInto(table, query->table); - if (uuid) - query->uuid = uuid->as()->value.get(); + query->database = table_id.database_name; + query->table = table_id.table_name; + query->uuid = table_id.uuid; query->cluster = cluster_str; - tryGetIdentifierNameInto(to_database, query->to_database); - tryGetIdentifierNameInto(to_table, query->to_table); - query->set(query->columns_list, columns_list); query->set(query->storage, storage); @@ -504,7 +483,6 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e ParserKeyword s_temporary("TEMPORARY"); ParserKeyword s_attach("ATTACH"); ParserKeyword s_if_not_exists("IF NOT EXISTS"); - ParserKeyword s_uuid("UUID"); ParserKeyword s_as("AS"); ParserKeyword s_view("VIEW"); ParserKeyword s_live("LIVE"); @@ -515,14 +493,10 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e ParserIdentifier name_p; ParserTablePropertiesDeclarationList table_properties_p; ParserSelectWithUnionQuery select_p; - ParserStringLiteral uuid_p; - ASTPtr database; - ASTPtr table; - ASTPtr uuid; + StorageID table_id; + StorageID to_table_id; ASTPtr columns_list; - ASTPtr to_database; - ASTPtr to_table; ASTPtr storage; ASTPtr as_database; ASTPtr as_table; @@ -555,22 +529,9 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e if (s_if_not_exists.ignore(pos, expected)) if_not_exists = true; - if (!name_p.parse(pos, table, expected)) + if (!parseStorageID(pos, table_id, expected)) return false; - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } - - if (attach && s_uuid.ignore(pos, expected)) - { - if (!uuid_p.parse(pos, uuid, expected)) - return false; - } - if (ParserKeyword{"ON"}.ignore(pos, expected)) { if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) @@ -580,15 +541,8 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e // TO [db.]table if (ParserKeyword{"TO"}.ignore(pos, expected)) { - if (!name_p.parse(pos, to_table, expected)) + if (!parseStorageID(pos, to_table_id, expected)) return false; - - if (s_dot.ignore(pos, expected)) - { - to_database = to_table; - if (!name_p.parse(pos, to_table, expected)) - return false; - } } /// Optional - a list of columns can be specified. It must fully comply with SELECT. @@ -617,14 +571,12 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e query->is_live_view = true; query->temporary = is_temporary; - tryGetIdentifierNameInto(database, query->database); - tryGetIdentifierNameInto(table, query->table); + query->database = table_id.database_name; + query->table = table_id.table_name; + query->uuid = table_id.uuid; query->cluster = cluster_str; - tryGetIdentifierNameInto(to_database, query->to_database); - tryGetIdentifierNameInto(to_table, query->to_table); - if (uuid) - query->uuid = uuid->as()->value.get(); + query->to_table_id = to_table_id; query->set(query->columns_list, columns_list); @@ -713,12 +665,9 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserNameList names_p; ParserStringLiteral uuid_p; - ASTPtr database; - ASTPtr table; - ASTPtr uuid; + StorageID table_id; + StorageID to_table_id; ASTPtr columns_list; - ASTPtr to_database; - ASTPtr to_table; ASTPtr storage; ASTPtr as_database; ASTPtr as_table; @@ -759,22 +708,9 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!replace_view && s_if_not_exists.ignore(pos, expected)) if_not_exists = true; - if (!name_p.parse(pos, table, expected)) + if (!parseStorageID(pos, table_id, expected)) return false; - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } - - if (attach && s_uuid.ignore(pos, expected)) - { - if (!uuid_p.parse(pos, uuid, expected)) - return false; - } - if (ParserKeyword{"ON"}.ignore(pos, expected)) { if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) @@ -784,15 +720,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec // TO [db.]table if (ParserKeyword{"TO"}.ignore(pos, expected)) { - if (!name_p.parse(pos, to_table, expected)) + if (!parseStorageID(pos, to_table_id, expected)) return false; - - if (s_dot.ignore(pos, expected)) - { - to_database = to_table; - if (!name_p.parse(pos, to_table, expected)) - return false; - } } /// Optional - a list of columns can be specified. It must fully comply with SELECT. @@ -805,7 +734,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec return false; } - if (is_materialized_view && !to_table) + if (is_materialized_view && to_table_id.empty()) { /// Internal ENGINE for MATERIALIZED VIEW must be specified. if (!storage_p.parse(pos, storage, expected)) @@ -833,14 +762,12 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->is_populate = is_populate; query->replace_view = replace_view; - tryGetIdentifierNameInto(database, query->database); - tryGetIdentifierNameInto(table, query->table); - if (uuid) - query->uuid = uuid->as()->value.get(); + query->database = table_id.database_name; + query->table = table_id.table_name; + query->uuid = table_id.uuid; query->cluster = cluster_str; - tryGetIdentifierNameInto(to_database, query->to_database); - tryGetIdentifierNameInto(to_table, query->to_table); + query->to_table_id = to_table_id; query->set(query->columns_list, columns_list); query->set(query->storage, storage); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index a51f26fa1aa..d859bc3c7e6 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -299,7 +299,7 @@ bool StorageKafka::checkDependencies(const StorageID & table_id) // Check the dependencies are ready? for (const auto & db_tab : dependencies) { - auto table = global_context.tryGetTable(db_tab.database_name, db_tab.table_name); //FIXME + auto table = global_context.tryGetTable(db_tab); if (!table) return false; @@ -351,7 +351,7 @@ void StorageKafka::threadFunc() bool StorageKafka::streamToViews() { auto table_id = getStorageID(); - auto table = global_context.getTable(table_id.database_name, table_id.table_name); + auto table = global_context.getTable(table_id); if (!table) throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 987996f9cc7..fe1fb21cc4b 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -14,7 +14,6 @@ limitations under the License. */ #include #include #include -#include #include #include #include @@ -33,10 +32,9 @@ limitations under the License. */ #include #include #include +#include #include -#include -#include #include #include @@ -52,42 +50,6 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name) -{ - auto db_and_table = getDatabaseAndTable(query, 0); - ASTPtr subquery = extractTableExpression(query, 0); - - if (!db_and_table && !subquery) - return; - - if (db_and_table) - { - select_table_name = db_and_table->table; - - if (db_and_table->database.empty()) - { - db_and_table->database = select_database_name; - AddDefaultDatabaseVisitor visitor(select_database_name); - visitor.visit(query); - } - else - select_database_name = db_and_table->database; - } - else if (auto * ast_select = subquery->as()) - { - if (ast_select->list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW); - - auto & inner_query = ast_select->list_of_selects->children.at(0); - - extractDependentTable(inner_query->as(), select_database_name, select_table_name); - } - else - throw Exception("Logical error while creating StorageLiveView." - " Could not retrieve table name from select query.", - DB::ErrorCodes::LOGICAL_ERROR); -} - void StorageLiveView::writeIntoLiveView( StorageLiveView & live_view, @@ -145,7 +107,7 @@ void StorageLiveView::writeIntoLiveView( if (!is_block_processed) { - auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); + auto parent_storage = context.getTable(live_view.getSelectTableID()); BlockInputStreams streams = {std::make_shared(block)}; auto proxy_storage = std::make_shared(parent_storage, std::move(streams), QueryProcessingStage::FetchColumns); InterpreterSelectQuery select_block(live_view.getInnerQuery(), @@ -177,7 +139,7 @@ void StorageLiveView::writeIntoLiveView( } } - auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); + auto parent_storage = context.getTable(live_view.getSelectTableID()); auto proxy_storage = std::make_shared(parent_storage, std::move(from), QueryProcessingStage::WithMergeableState); InterpreterSelectQuery select(live_view.getInnerQuery(), context, proxy_storage, QueryProcessingStage::Complete); BlockInputStreamPtr data = std::make_shared(select.execute().in); @@ -205,25 +167,19 @@ StorageLiveView::StorageLiveView( throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); /// Default value, if only table name exist in the query - select_database_name = local_context.getCurrentDatabase(); if (query.select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW); inner_query = query.select->list_of_selects->children.at(0); - ASTSelectQuery & select_query = typeid_cast(*inner_query); - extractDependentTable(select_query, select_database_name, select_table_name); + select_table_id = extractDependentTableFromSelectQuery(select_query, global_context, true); /// If the table is not specified - use the table `system.one` - if (select_table_name.empty()) - { - select_database_name = "system"; - select_table_name = "one"; - } + //FIXME why? + if (select_table_id.empty()) + select_table_id = StorageID("system", "one"); - global_context.addDependency( - StorageID(select_database_name, select_table_name), - table_id_); //FIXME + global_context.addDependency(select_table_id, table_id_); is_temporary = query.temporary; temporary_live_view_timeout = local_context.getSettingsRef().temporary_live_view_timeout.totalSeconds(); @@ -255,7 +211,7 @@ Block StorageLiveView::getHeader() const if (!sample_block) { - auto storage = global_context.getTable(select_database_name, select_table_name); + auto storage = global_context.getTable(select_table_id); sample_block = InterpreterSelectQuery(inner_query, global_context, storage, SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock(); sample_block.insert({DataTypeUInt64().createColumnConst( @@ -290,7 +246,7 @@ bool StorageLiveView::getNewBlocks() mergeable_blocks = std::make_shared>(); mergeable_blocks->push_back(new_mergeable_blocks); BlockInputStreamPtr from = std::make_shared(std::make_shared(new_mergeable_blocks), mergeable_stream->getHeader()); - auto proxy_storage = ProxyStorage::createProxyStorage(global_context.getTable(select_database_name, select_table_name), {from}, QueryProcessingStage::WithMergeableState); + auto proxy_storage = ProxyStorage::createProxyStorage(global_context.getTable(select_table_id), {from}, QueryProcessingStage::WithMergeableState); InterpreterSelectQuery select(inner_query->clone(), global_context, proxy_storage, SelectQueryOptions(QueryProcessingStage::Complete)); BlockInputStreamPtr data = std::make_shared(select.execute().in); @@ -375,7 +331,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co if (drop_table) { - if (storage->global_context.tryGetTable(table_id.database_name, table_id.table_name)) //FIXME + if (storage->global_context.tryGetTable(table_id)) { try { @@ -468,9 +424,7 @@ StorageLiveView::~StorageLiveView() void StorageLiveView::drop(TableStructureWriteLockHolder &) { auto table_id = getStorageID(); - global_context.removeDependency( - StorageID(select_database_name, select_table_name), - table_id); //FIXME + global_context.removeDependency(select_table_id, table_id); std::lock_guard lock(mutex); is_dropped = true; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index 63a923df2cc..9c495c4b1d4 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -41,8 +41,7 @@ friend class LiveViewBlockOutputStream; public: ~StorageLiveView() override; String getName() const override { return "LiveView"; } - String getSelectDatabaseName() const { return select_database_name; } - String getSelectTableName() const { return select_table_name; } + StorageID getSelectTableID() const { return select_table_id; } NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; @@ -140,8 +139,7 @@ public: const Context & context); private: - String select_database_name; - String select_table_name; + StorageID select_table_id; ASTPtr inner_query; Context & global_context; bool is_temporary = false; diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h index cdce84729ff..23ff37e0e11 100644 --- a/dbms/src/Storages/StorageID.h +++ b/dbms/src/Storages/StorageID.h @@ -11,28 +11,36 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; + struct StorageID { String database_name; String table_name; String uuid; - StorageID() = delete; + //StorageID() = delete; + StorageID() = default; + + //TODO StorageID(const ASTPtr & query_with_one_table, const Context & context) to get db and table names (and maybe uuid) from query + //But there are a lot of different ASTs with db and table name + //And it looks like it depends on https://github.com/ClickHouse/ClickHouse/pull/7774 StorageID(const String & database, const String & table, const String & uuid_ = {}) : database_name(database), table_name(table), uuid(uuid_) { - assert_not_empty(); } String getFullTableName() const { + assert_valid(); return (database_name.empty() ? "" : database_name + ".") + table_name; } String getNameForLogs() const { - return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + " (UUID " + uuid + ")"; + assert_valid(); + return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + (uuid.empty() ? "" : " (UUID " + uuid + ")"); } String getId() const @@ -45,13 +53,31 @@ struct StorageID bool operator<(const StorageID & rhs) const { - return std::tie(uuid, database_name, table_name) < std::tie(rhs.uuid, rhs.database_name, rhs.table_name); + assert_valid(); + /// It's needed for ViewDependencies + if (uuid.empty() && rhs.uuid.empty()) + /// If both IDs don't have UUID, compare them like pair of strings + return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name); + else if (!uuid.empty() && !rhs.uuid.empty()) + /// If both IDs have UUID, compare UUIDs and ignore database and table name + return uuid < rhs.uuid; + else + /// All IDs without UUID are less, then all IDs with UUID + return uuid.empty(); } - void assert_not_empty() const + bool empty() const { - if (database_name.empty() && table_name.empty()) + return table_name.empty() || (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && uuid.empty()); + } + + void assert_valid() const + { + if (empty()) throw Exception("empty table name", ErrorCodes::LOGICAL_ERROR); + if (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && uuid.empty() && !database_name.empty()) + throw Exception("unexpected database name", ErrorCodes::LOGICAL_ERROR); + } }; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 9ba30a7d108..b2e4df4cf7d 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -27,6 +27,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int INCORRECT_QUERY; extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW; + extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW; } static inline String generateInnerTableName(const String & table_name) @@ -34,39 +35,37 @@ static inline String generateInnerTableName(const String & table_name) return ".inner." + table_name; } -static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name) +StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, Context & context, bool is_live_view /*= false*/, bool need_visitor /*= true*/) { + if (need_visitor) + { + AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr); + visitor.visit(query); + } auto db_and_table = getDatabaseAndTable(query, 0); ASTPtr subquery = extractTableExpression(query, 0); if (!db_and_table && !subquery) - return; + return {}; //FIXME in which cases we cannot get table name? if (db_and_table) { - select_table_name = db_and_table->table; - - if (db_and_table->database.empty()) - { - db_and_table->database = select_database_name; - AddDefaultDatabaseVisitor visitor(select_database_name); - visitor.visit(query); - } - else - select_database_name = db_and_table->database; + //TODO uuid + return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/); } else if (auto * ast_select = subquery->as()) { if (ast_select->list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + throw Exception(String("UNION is not supported for ") + (is_live_view ? "LIVE VIEW" : "MATERIALIZED VIEW"), + is_live_view ? ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW : ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); auto & inner_query = ast_select->list_of_selects->children.at(0); - extractDependentTable(inner_query->as(), select_database_name, select_table_name); + return extractDependentTableFromSelectQuery(inner_query->as(), context, is_live_view, false); } else - throw Exception("Logical error while creating StorageMaterializedView." - " Could not retrieve table name from select query.", + throw Exception(String("Logical error while creating Storage") + (is_live_view ? "Live" : "Materialized") + + "View. Could not retrieve table name from select query.", DB::ErrorCodes::LOGICAL_ERROR); } @@ -105,47 +104,36 @@ StorageMaterializedView::StorageMaterializedView( if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); - if (!query.storage && query.to_table.empty()) + /// If the destination table is not set, use inner table + has_inner_table = query.to_table_id.empty(); + if (has_inner_table && !query.storage) throw Exception( "You must specify where to save results of a MaterializedView query: either ENGINE or an existing table in a TO clause", ErrorCodes::INCORRECT_QUERY); - /// Default value, if only table name exist in the query - select_database_name = local_context.getCurrentDatabase(); if (query.select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); inner_query = query.select->list_of_selects->children.at(0); auto & select_query = inner_query->as(); - extractDependentTable(select_query, select_database_name, select_table_name); + select_table_id = extractDependentTableFromSelectQuery(select_query, local_context); checkAllowedQueries(select_query); - if (!select_table_name.empty()) - global_context.addDependency( - StorageID(select_database_name, select_table_name), - table_id_); //FIXME - - // If the destination table is not set, use inner table - if (!query.to_table.empty()) + if (!has_inner_table) + target_table_id = query.to_table_id; + else if (attach_) { - target_database_name = query.to_database; - target_table_name = query.to_table; + /// If there is an ATTACH request, then the internal table must already be created. + //TODO use uuid + target_table_id = StorageID(table_id_.database_name, generateInnerTableName(table_id_.table_name)); } else - { - target_database_name = table_id_.database_name; - target_table_name = generateInnerTableName(table_id_.table_name); - has_inner_table = true; - } - - /// If there is an ATTACH request, then the internal table must already be connected. - if (!attach_ && has_inner_table) { /// We will create a query to create an internal table. auto manual_create_query = std::make_shared(); - manual_create_query->database = target_database_name; - manual_create_query->table = target_table_name; + manual_create_query->database = table_id_.database_name; + manual_create_query->table = generateInnerTableName(table_id_.table_name); auto new_columns_list = std::make_shared(); new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr()); @@ -153,24 +141,15 @@ StorageMaterializedView::StorageMaterializedView( manual_create_query->set(manual_create_query->columns_list, new_columns_list); manual_create_query->set(manual_create_query->storage, query.storage->ptr()); - /// Execute the query. - try - { - InterpreterCreateQuery create_interpreter(manual_create_query, local_context); - create_interpreter.setInternal(true); - create_interpreter.execute(); - } - catch (...) - { - /// In case of any error we should remove dependency to the view. - if (!select_table_name.empty()) - global_context.removeDependency( - StorageID(select_database_name, select_table_name), - table_id_); //FIXME + InterpreterCreateQuery create_interpreter(manual_create_query, local_context); + create_interpreter.setInternal(true); + create_interpreter.execute(); - throw; - } + target_table_id = global_context.getTable(manual_create_query->database, manual_create_query->table)->getStorageID(); } + + if (!select_table_id.empty()) + global_context.addDependency(select_table_id, table_id_); } NameAndTypePair StorageMaterializedView::getColumn(const String & column_name) const @@ -214,14 +193,14 @@ BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const } -static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const String & target_database_name, const String & target_table_name) +static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const StorageID & target_table_id) { - if (global_context.tryGetTable(target_database_name, target_table_name)) + if (global_context.tryGetTable(target_table_id)) { /// We create and execute `drop` query for internal table. auto drop_query = std::make_shared(); - drop_query->database = target_database_name; - drop_query->table = target_table_name; + drop_query->database = target_table_id.database_name; + drop_query->table = target_table_id.table_name; drop_query->kind = kind; ASTPtr ast_drop_query = drop_query; InterpreterDropQuery drop_interpreter(ast_drop_query, global_context); @@ -233,25 +212,23 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, void StorageMaterializedView::drop(TableStructureWriteLockHolder &) { auto table_id = getStorageID(); - global_context.removeDependency( - StorageID(select_database_name, select_table_name), - table_id); //FIXME + global_context.removeDependency(select_table_id, table_id); if (has_inner_table && tryGetTargetTable()) - executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name); + executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id); } void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { if (has_inner_table) - executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name); + executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_table_id); } void StorageMaterializedView::checkStatementCanBeForwarded() const { if (!has_inner_table) throw Exception( - "MATERIALIZED VIEW targets existing table " + target_database_name + "." + target_table_name + ". " + "MATERIALIZED VIEW targets existing table " + target_table_id.getNameForLogs() + ". " + "Execute the statement directly on it.", ErrorCodes::INCORRECT_QUERY); } @@ -273,74 +250,59 @@ void StorageMaterializedView::mutate(const MutationCommands & commands, const Co getTargetTable()->mutate(commands, context); } -static void executeRenameQuery(Context & global_context, const String & database_name, const String & table_original_name, const String & new_table_name) -{ - if (global_context.tryGetTable(database_name, table_original_name)) - { - auto rename = std::make_shared(); - - ASTRenameQuery::Table from; - from.database = database_name; - from.table = table_original_name; - - ASTRenameQuery::Table to; - to.database = database_name; - to.table = new_table_name; - - ASTRenameQuery::Element elem; - elem.from = from; - elem.to = to; - - rename->elements.emplace_back(elem); - - InterpreterRenameQuery(rename, global_context).execute(); - } -} - - void StorageMaterializedView::renameInMemory(const String & new_database_name, const String & new_table_name, std::unique_lock *) { //FIXME if (has_inner_table && tryGetTargetTable()) { - String new_target_table_name = generateInnerTableName(new_table_name); - executeRenameQuery(global_context, target_database_name, target_table_name, new_target_table_name); - target_table_name = new_target_table_name; + auto new_target_table_name = generateInnerTableName(new_table_name); + auto rename = std::make_shared(); + + ASTRenameQuery::Table from; + from.database = target_table_id.database_name; + from.table = target_table_id.table_name; + + ASTRenameQuery::Table to; + to.database = target_table_id.database_name; + to.table = new_target_table_name; + + ASTRenameQuery::Element elem; + elem.from = from; + elem.to = to; + rename->elements.emplace_back(elem); + + InterpreterRenameQuery(rename, global_context).execute(); + target_table_id.table_name = new_target_table_name; } auto lock = global_context.getLock(); std::unique_lock name_lock; auto table_id = getStorageID(&name_lock); - global_context.removeDependencyUnsafe( - StorageID(select_database_name, select_table_name), - table_id); + global_context.removeDependencyUnsafe(select_table_id, table_id); IStorage::renameInMemory(new_database_name, new_table_name, &name_lock); - global_context.addDependencyUnsafe( - StorageID(select_database_name, select_table_name), - table_id); + auto new_table_id = getStorageID(&name_lock); + global_context.addDependencyUnsafe(select_table_id, new_table_id); } void StorageMaterializedView::shutdown() { auto table_id = getStorageID(); /// Make sure the dependency is removed after DETACH TABLE - global_context.removeDependency( - StorageID(select_database_name, select_table_name), - table_id); //FIXME + global_context.removeDependency(select_table_id, table_id); } StoragePtr StorageMaterializedView::getTargetTable() const { - return global_context.getTable(target_database_name, target_table_name); + return global_context.getTable(target_table_id); } StoragePtr StorageMaterializedView::tryGetTargetTable() const { - return global_context.tryGetTable(target_database_name, target_table_name); + return global_context.tryGetTable(target_table_id); } Strings StorageMaterializedView::getDataPaths() const diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index a86f49cd0f6..19055d41011 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -9,6 +9,9 @@ namespace DB { +StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, Context & context, bool is_live_view = false, bool need_visitor = true); + + class StorageMaterializedView : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; @@ -66,10 +69,8 @@ public: Strings getDataPaths() const override; private: - String select_database_name; - String select_table_name; - String target_database_name; - String target_table_name; + StorageID select_table_id; + StorageID target_table_id; ASTPtr inner_query; Context & global_context; bool has_inner_table = false;