thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

use StorageID in views

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-12-10 22:48:16 +03:00
parent 03b1a576ba
commit 0b70bffe36
18 changed files with 229 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -46,7 +46,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
for (const auto & database_table : dependencies)
{
auto dependent_table = context.getTable(database_table.database_name, database_table.table_name); //FIXME
auto dependent_table = context.getTable(database_table);
ASTPtr query;
BlockOutputStreamPtr out;
@ -219,7 +219,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
/// InterpreterSelectQuery will do processing of alias columns.
Context local_context = *views_context;
local_context.addViewSource(
StorageValues::create(StorageID(storage->getDatabaseName(), storage->getTableName()), storage->getColumns(), //FIXME
StorageValues::create(storage->getStorageID(), storage->getColumns(),
block));
select.emplace(view.query, local_context, SelectQueryOptions());
in = std::make_shared<MaterializingBlockInputStream>(select->execute().in);

1
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -29,7 +29,6 @@ namespace DB
{
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_";
namespace ErrorCodes
{

34
dbms/src/Interpreters/Context.cpp
View File

@ -728,7 +728,6 @@ void Context::checkDatabaseAccessRightsImpl(const std::string & database_name) c
throw Exception("Access denied to database " + database_name + " for user " + client_info.current_user , ErrorCodes::DATABASE_ACCESS_DENIED);
}
//FIXME use uuids if not empty
void Context::addDependencyUnsafe(const StorageID & from, const StorageID & where)
{
@ -737,7 +736,7 @@ void Context::addDependencyUnsafe(const StorageID & from, const StorageID & wher
shared->view_dependencies[from].insert(where);
// Notify table of dependencies change
auto table = tryGetTable(from.database_name, from.table_name);
auto table = tryGetTable(from);
if (table != nullptr)
table->updateDependencies();
}
@ -755,7 +754,7 @@ void Context::removeDependencyUnsafe(const StorageID & from, const StorageID & w
shared->view_dependencies[from].erase(where);
// Notify table of dependencies change
auto table = tryGetTable(from.database_name, from.table_name);
auto table = tryGetTable(from);
if (table != nullptr)
table->updateDependencies();
}
@ -910,24 +909,32 @@ StoragePtr Context::tryGetExternalTable(const String & table_name) const
return jt->second.first;
}
StoragePtr Context::getTable(const String & database_name, const String & table_name) const
{
return getTable(StorageID(database_name, table_name));
}
StoragePtr Context::getTable(const StorageID & table_id) const
{
Exception exc;
auto res = getTableImpl(database_name, table_name, &exc);
auto res = getTableImpl(table_id, &exc);
if (!res)
throw exc;
return res;
}
StoragePtr Context::tryGetTable(const String & database_name, const String & table_name) const
{
return getTableImpl(database_name, table_name, nullptr);
return tryGetTable(StorageID(database_name, table_name));
}
StoragePtr Context::tryGetTable(const StorageID & table_id) const
{
return getTableImpl(table_id, nullptr);
}
StoragePtr Context::getTableImpl(const String & database_name, const String & table_name, Exception * exception) const
StoragePtr Context::getTableImpl(const StorageID & table_id, Exception * exception) const
{
String db;
DatabasePtr database;
@ -935,14 +942,15 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta
{
auto lock = getLock();
if (database_name.empty())
if (table_id.database_name.empty())
{
StoragePtr res = tryGetExternalTable(table_name);
StoragePtr res = tryGetExternalTable(table_id.table_name);
if (res)
return res;
}
db = resolveDatabase(database_name, current_database);
//FIXME what if table was moved to another database?
db = resolveDatabase(table_id.database_name, current_database);
checkDatabaseAccessRightsImpl(db);
Databases::const_iterator it = shared->databases.find(db);
@ -956,11 +964,11 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta
database = it->second;
}
auto table = database->tryGetTable(*this, table_name);
auto table = database->tryGetTable(*this, table_id.table_name);
if (!table)
{
if (exception)
*exception = Exception("Table " + backQuoteIfNeed(db) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
*exception = Exception("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
return {};
}

9
dbms/src/Interpreters/Context.h
View File

@ -88,10 +88,6 @@ class CompiledExpressionCache;
#endif
/// (database name, table name)
//FIXME replace with StorageID
//using DatabaseAndTableName = std::pair<String, String>;
/// Table -> set of table-views that make SELECT from it.
using ViewDependencies = std::map<StorageID, std::set<StorageID>>;
using Dependencies = std::vector<StorageID>;
@ -125,6 +121,7 @@ using IHostContextPtr = std::shared_ptr<IHostContext>;
*
* Everything is encapsulated for all sorts of checks and locks.
*/
///TODO remove syntax sugar and legacy methods from Context (e.g. getInputFormat(...) which just returns object from factory)
class Context
{
private:
@ -276,7 +273,9 @@ public:
Tables getExternalTables() const;
StoragePtr tryGetExternalTable(const String & table_name) const;
StoragePtr getTable(const String & database_name, const String & table_name) const;
StoragePtr getTable(const StorageID & table_id) const;
StoragePtr tryGetTable(const String & database_name, const String & table_name) const;
StoragePtr tryGetTable(const StorageID & table_id) const;
void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {});
void addScalar(const String & name, const Block & block);
bool hasScalar(const String & name) const;
@ -578,7 +577,7 @@ private:
EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const;
StoragePtr getTableImpl(const String & database_name, const String & table_name, Exception * exception) const;
StoragePtr getTableImpl(const StorageID & table_id, Exception * exception) const;
SessionKey getSessionKey(const String & session_id) const;

8
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -548,8 +548,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
if (!create.cluster.empty())
{
NameSet databases{create.database};
if (!create.to_table.empty())
databases.emplace(create.to_database);
if (!create.to_table_id.empty())
databases.emplace(create.to_table_id.database_name);
/// NOTE: if it's CREATE query and create.database is DatabaseAtomic, different UUIDs will be generated on all servers.
/// However, it allows to use UUID as replica name.
@ -577,8 +577,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
String current_database = context.getCurrentDatabase();
if (create.database.empty())
create.database = current_database;
if (create.to_database.empty())
create.to_database = current_database;
if (!create.to_table_id.empty() && create.to_table_id.database_name.empty())
create.to_table_id.database_name = current_database;
if (create.select && (create.is_view || create.is_materialized_view || create.is_live_view))
{

1
dbms/src/Parsers/ASTAlterQuery.h
View File

@ -147,6 +147,7 @@ public:
String with_name;
/// REPLACE(ATTACH) PARTITION partition FROM db.table
//FIXME use StorageID
String from_database;
String from_table;
/// To distinguish REPLACE and ATTACH PARTITION partition FROM db.table

13
dbms/src/Parsers/ASTCreateQuery.cpp
View File

@ -233,9 +233,11 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
<< what << " "
<< (if_not_exists ? "IF NOT EXISTS " : "")
<< (settings.hilite ? hilite_none : "")
<< (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table)
<< (!uuid.empty() ? " UUID " + quoteString(uuid) : "");
formatOnCluster(settings);
<< (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table);
if (!uuid.empty())
settings.ostr << (settings.hilite ? hilite_keyword : "") << " UUID " << (settings.hilite ? hilite_none : "")
<< quoteString(uuid);
formatOnCluster(settings);
}
else
{
@ -250,11 +252,12 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
as_table_function->formatImpl(settings, state, frame);
}
if (!to_table.empty())
if (!to_table_id.empty())
{
settings.ostr
<< (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : "")
<< (!to_database.empty() ? backQuoteIfNeed(to_database) + "." : "") << backQuoteIfNeed(to_table);
<< (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "")
<< backQuoteIfNeed(to_table_id.table_name);
}
if (!as_table.empty())

5
dbms/src/Parsers/ASTCreateQuery.h
View File

@ -4,6 +4,7 @@
#include <Parsers/ASTQueryWithOnCluster.h>
#include <Parsers/ASTDictionary.h>
#include <Parsers/ASTDictionaryAttributeDeclaration.h>
#include <Storages/StorageID.h>
namespace DB
@ -65,15 +66,13 @@ public:
ASTColumns * columns_list = nullptr;
ASTExpressionList * dictionary_attributes_list = nullptr; /// attributes of dictionary
ASTExpressionList * tables = nullptr;
String to_database; /// For CREATE MATERIALIZED VIEW mv TO table.
String to_table;
StorageID to_table_id; /// For CREATE MATERIALIZED VIEW mv TO table.
ASTStorage * storage = nullptr;
String as_database;
String as_table;
ASTPtr as_table_function;
ASTSelectWithUnionQuery * select = nullptr;
ASTDictionary * dictionary = nullptr; /// dictionary definition (layout, primary key, etc.)
String uuid; /// For ATTACH TABLE query when db engine is Atomic
/** Get the text that identifies this element. */
String getID(char delim) const override { return (attach ? "AttachQuery" : "CreateQuery") + (delim + database) + delim + table; }

1
dbms/src/Parsers/ASTQueryWithTableAndOutput.h
View File

@ -15,6 +15,7 @@ class ASTQueryWithTableAndOutput : public ASTQueryWithOutput
public:
String database;
String table;
String uuid;
bool temporary{false};
protected:

36
dbms/src/Parsers/ExpressionElementParsers.cpp
View File

@ -29,7 +29,7 @@
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/queryToString.h>
#include <boost/algorithm/string.hpp>
#include <Storages/StorageID.h>
#include "ASTColumnsMatcher.h"
@ -197,6 +197,40 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
}
bool parseStorageID(IParser::Pos & pos, StorageID & res, Expected & expected)
{
ParserKeyword s_uuid("UUID");
ParserIdentifier name_p;
ParserStringLiteral uuid_p;
ParserToken s_dot(TokenType::Dot);
ASTPtr database;
ASTPtr table;
ASTPtr uuid;
if (!name_p.parse(pos, table, expected))
return false;
if (s_dot.ignore(pos, expected))
{
database = table;
if (!name_p.parse(pos, table, expected))
return false;
}
if (s_uuid.ignore(pos, expected))
{
if (!uuid_p.parse(pos, uuid, expected))
return false;
}
tryGetIdentifierNameInto(database, res.database_name);
tryGetIdentifierNameInto(table, res.table_name);
res.uuid = uuid ? uuid->as<ASTLiteral>()->value.get<String>() : "";
return true;
}
bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ParserIdentifier id_parser;

6
dbms/src/Parsers/ExpressionElementParsers.h
View File

@ -56,6 +56,12 @@ protected:
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
};
struct StorageID;
/// Table name, possibly with database name and UUID as string literal
/// [db_name.]table_name [UUID 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx']
//TODO replace with class
bool parseStorageID(IParser::Pos & pos, StorageID & res, Expected & expected);
/// Just *
class ParserAsterisk : public IParserBase
{

125
dbms/src/Parsers/ParserCreateQuery.cpp
View File

@ -13,6 +13,7 @@
#include <Parsers/ASTConstraintDeclaration.h>
#include <Parsers/ParserDictionary.h>
#include <Parsers/ParserDictionaryAttributeDeclaration.h>
#include <Storages/StorageID.h>
namespace DB
@ -340,7 +341,6 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
ParserKeyword s_temporary("TEMPORARY");
ParserKeyword s_table("TABLE");
ParserKeyword s_if_not_exists("IF NOT EXISTS");
ParserKeyword s_uuid("UUID");
ParserKeyword s_on("ON");
ParserKeyword s_as("AS");
ParserToken s_dot(TokenType::Dot);
@ -352,14 +352,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
ParserSelectWithUnionQuery select_p;
ParserFunction table_function_p;
ParserNameList names_p;
ParserStringLiteral uuid_p;
ASTPtr database;
ASTPtr table;
ASTPtr uuid;
StorageID table_id;
ASTPtr columns_list;
ASTPtr to_database;
ASTPtr to_table;
ASTPtr storage;
ASTPtr as_database;
ASTPtr as_table;
@ -389,22 +384,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
if (s_if_not_exists.ignore(pos, expected))
if_not_exists = true;
if (!name_p.parse(pos, table, expected))
if (!parseStorageID(pos, table_id, expected))
return false;
if (s_dot.ignore(pos, expected))
{
database = table;
if (!name_p.parse(pos, table, expected))
return false;
}
if (attach && s_uuid.ignore(pos, expected))
{
if (!uuid_p.parse(pos, uuid, expected))
return false;
}
if (s_on.ignore(pos, expected))
{
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
@ -421,8 +403,9 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
query->if_not_exists = if_not_exists;
query->cluster = cluster_str;
tryGetIdentifierNameInto(database, query->database);
tryGetIdentifierNameInto(table, query->table);
query->database = table_id.database_name;
query->table = table_id.table_name;
query->uuid = table_id.uuid;
return true;
}
@ -479,15 +462,11 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
query->if_not_exists = if_not_exists;
query->temporary = is_temporary;
tryGetIdentifierNameInto(database, query->database);
tryGetIdentifierNameInto(table, query->table);
if (uuid)
query->uuid = uuid->as<ASTLiteral>()->value.get<String>();
query->database = table_id.database_name;
query->table = table_id.table_name;
query->uuid = table_id.uuid;
query->cluster = cluster_str;
tryGetIdentifierNameInto(to_database, query->to_database);
tryGetIdentifierNameInto(to_table, query->to_table);
query->set(query->columns_list, columns_list);
query->set(query->storage, storage);
@ -504,7 +483,6 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
ParserKeyword s_temporary("TEMPORARY");
ParserKeyword s_attach("ATTACH");
ParserKeyword s_if_not_exists("IF NOT EXISTS");
ParserKeyword s_uuid("UUID");
ParserKeyword s_as("AS");
ParserKeyword s_view("VIEW");
ParserKeyword s_live("LIVE");
@ -515,14 +493,10 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
ParserIdentifier name_p;
ParserTablePropertiesDeclarationList table_properties_p;
ParserSelectWithUnionQuery select_p;
ParserStringLiteral uuid_p;
ASTPtr database;
ASTPtr table;
ASTPtr uuid;
StorageID table_id;
StorageID to_table_id;
ASTPtr columns_list;
ASTPtr to_database;
ASTPtr to_table;
ASTPtr storage;
ASTPtr as_database;
ASTPtr as_table;
@ -555,22 +529,9 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
if (s_if_not_exists.ignore(pos, expected))
if_not_exists = true;
if (!name_p.parse(pos, table, expected))
if (!parseStorageID(pos, table_id, expected))
return false;
if (s_dot.ignore(pos, expected))
{
database = table;
if (!name_p.parse(pos, table, expected))
return false;
}
if (attach && s_uuid.ignore(pos, expected))
{
if (!uuid_p.parse(pos, uuid, expected))
return false;
}
if (ParserKeyword{"ON"}.ignore(pos, expected))
{
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
@ -580,15 +541,8 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
// TO [db.]table
if (ParserKeyword{"TO"}.ignore(pos, expected))
{
if (!name_p.parse(pos, to_table, expected))
if (!parseStorageID(pos, to_table_id, expected))
return false;
if (s_dot.ignore(pos, expected))
{
to_database = to_table;
if (!name_p.parse(pos, to_table, expected))
return false;
}
}
/// Optional - a list of columns can be specified. It must fully comply with SELECT.
@ -617,14 +571,12 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
query->is_live_view = true;
query->temporary = is_temporary;
tryGetIdentifierNameInto(database, query->database);
tryGetIdentifierNameInto(table, query->table);
query->database = table_id.database_name;
query->table = table_id.table_name;
query->uuid = table_id.uuid;
query->cluster = cluster_str;
tryGetIdentifierNameInto(to_database, query->to_database);
tryGetIdentifierNameInto(to_table, query->to_table);
if (uuid)
query->uuid = uuid->as<ASTLiteral>()->value.get<String>();
query->to_table_id = to_table_id;
query->set(query->columns_list, columns_list);
@ -713,12 +665,9 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
ParserNameList names_p;
ParserStringLiteral uuid_p;
ASTPtr database;
ASTPtr table;
ASTPtr uuid;
StorageID table_id;
StorageID to_table_id;
ASTPtr columns_list;
ASTPtr to_database;
ASTPtr to_table;
ASTPtr storage;
ASTPtr as_database;
ASTPtr as_table;
@ -759,22 +708,9 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
if (!replace_view && s_if_not_exists.ignore(pos, expected))
if_not_exists = true;
if (!name_p.parse(pos, table, expected))
if (!parseStorageID(pos, table_id, expected))
return false;
if (s_dot.ignore(pos, expected))
{
database = table;
if (!name_p.parse(pos, table, expected))
return false;
}
if (attach && s_uuid.ignore(pos, expected))
{
if (!uuid_p.parse(pos, uuid, expected))
return false;
}
if (ParserKeyword{"ON"}.ignore(pos, expected))
{
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
@ -784,15 +720,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
// TO [db.]table
if (ParserKeyword{"TO"}.ignore(pos, expected))
{
if (!name_p.parse(pos, to_table, expected))
if (!parseStorageID(pos, to_table_id, expected))
return false;
if (s_dot.ignore(pos, expected))
{
to_database = to_table;
if (!name_p.parse(pos, to_table, expected))
return false;
}
}
/// Optional - a list of columns can be specified. It must fully comply with SELECT.
@ -805,7 +734,7 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
return false;
}
if (is_materialized_view && !to_table)
if (is_materialized_view && to_table_id.empty())
{
/// Internal ENGINE for MATERIALIZED VIEW must be specified.
if (!storage_p.parse(pos, storage, expected))
@ -833,14 +762,12 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
query->is_populate = is_populate;
query->replace_view = replace_view;
tryGetIdentifierNameInto(database, query->database);
tryGetIdentifierNameInto(table, query->table);
if (uuid)
query->uuid = uuid->as<ASTLiteral>()->value.get<String>();
query->database = table_id.database_name;
query->table = table_id.table_name;
query->uuid = table_id.uuid;
query->cluster = cluster_str;
tryGetIdentifierNameInto(to_database, query->to_database);
tryGetIdentifierNameInto(to_table, query->to_table);
query->to_table_id = to_table_id;
query->set(query->columns_list, columns_list);
query->set(query->storage, storage);

4
dbms/src/Storages/Kafka/StorageKafka.cpp
View File

@ -299,7 +299,7 @@ bool StorageKafka::checkDependencies(const StorageID & table_id)
// Check the dependencies are ready?
for (const auto & db_tab : dependencies)
{
auto table = global_context.tryGetTable(db_tab.database_name, db_tab.table_name); //FIXME
auto table = global_context.tryGetTable(db_tab);
if (!table)
return false;
@ -351,7 +351,7 @@ void StorageKafka::threadFunc()
bool StorageKafka::streamToViews()
{
auto table_id = getStorageID();
auto table = global_context.getTable(table_id.database_name, table_id.table_name);
auto table = global_context.getTable(table_id);
if (!table)
throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR);

70
dbms/src/Storages/LiveView/StorageLiveView.cpp
View File

@ -14,7 +14,6 @@ limitations under the License. */
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTWatchQuery.h>
#include <Parsers/ASTDropQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterDropQuery.h>
@ -33,10 +32,9 @@ limitations under the License. */
#include <Storages/LiveView/LiveViewBlockOutputStream.h>
#include <Storages/LiveView/LiveViewEventsBlockInputStream.h>
#include <Storages/LiveView/ProxyStorage.h>
#include <Storages/StorageMaterializedView.h>
#include <Storages/StorageFactory.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Interpreters/DatabaseAndTableWithAlias.h>
#include <Interpreters/AddDefaultDatabaseVisitor.h>
@ -52,42 +50,6 @@ namespace ErrorCodes
extern const int SUPPORT_IS_DISABLED;
}
static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name)
{
auto db_and_table = getDatabaseAndTable(query, 0);
ASTPtr subquery = extractTableExpression(query, 0);
if (!db_and_table && !subquery)
return;
if (db_and_table)
{
select_table_name = db_and_table->table;
if (db_and_table->database.empty())
{
db_and_table->database = select_database_name;
AddDefaultDatabaseVisitor visitor(select_database_name);
visitor.visit(query);
}
else
select_database_name = db_and_table->database;
}
else if (auto * ast_select = subquery->as<ASTSelectWithUnionQuery>())
{
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
auto & inner_query = ast_select->list_of_selects->children.at(0);
extractDependentTable(inner_query->as<ASTSelectQuery &>(), select_database_name, select_table_name);
}
else
throw Exception("Logical error while creating StorageLiveView."
" Could not retrieve table name from select query.",
DB::ErrorCodes::LOGICAL_ERROR);
}
void StorageLiveView::writeIntoLiveView(
StorageLiveView & live_view,
@ -145,7 +107,7 @@ void StorageLiveView::writeIntoLiveView(
if (!is_block_processed)
{
auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName());
auto parent_storage = context.getTable(live_view.getSelectTableID());
BlockInputStreams streams = {std::make_shared<OneBlockInputStream>(block)};
auto proxy_storage = std::make_shared<ProxyStorage>(parent_storage, std::move(streams), QueryProcessingStage::FetchColumns);
InterpreterSelectQuery select_block(live_view.getInnerQuery(),
@ -177,7 +139,7 @@ void StorageLiveView::writeIntoLiveView(
}
}
auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName());
auto parent_storage = context.getTable(live_view.getSelectTableID());
auto proxy_storage = std::make_shared<ProxyStorage>(parent_storage, std::move(from), QueryProcessingStage::WithMergeableState);
InterpreterSelectQuery select(live_view.getInnerQuery(), context, proxy_storage, QueryProcessingStage::Complete);
BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
@ -205,25 +167,19 @@ StorageLiveView::StorageLiveView(
throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
/// Default value, if only table name exist in the query
select_database_name = local_context.getCurrentDatabase();
if (query.select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
inner_query = query.select->list_of_selects->children.at(0);
ASTSelectQuery & select_query = typeid_cast<ASTSelectQuery &>(*inner_query);
extractDependentTable(select_query, select_database_name, select_table_name);
select_table_id = extractDependentTableFromSelectQuery(select_query, global_context, true);
/// If the table is not specified - use the table `system.one`
if (select_table_name.empty())
{
select_database_name = "system";
select_table_name = "one";
}
//FIXME why?
if (select_table_id.empty())
select_table_id = StorageID("system", "one");
global_context.addDependency(
StorageID(select_database_name, select_table_name),
table_id_); //FIXME
global_context.addDependency(select_table_id, table_id_);
is_temporary = query.temporary;
temporary_live_view_timeout = local_context.getSettingsRef().temporary_live_view_timeout.totalSeconds();
@ -255,7 +211,7 @@ Block StorageLiveView::getHeader() const
if (!sample_block)
{
auto storage = global_context.getTable(select_database_name, select_table_name);
auto storage = global_context.getTable(select_table_id);
sample_block = InterpreterSelectQuery(inner_query, global_context, storage,
SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock();
sample_block.insert({DataTypeUInt64().createColumnConst(
@ -290,7 +246,7 @@ bool StorageLiveView::getNewBlocks()
mergeable_blocks = std::make_shared<std::vector<BlocksPtr>>();
mergeable_blocks->push_back(new_mergeable_blocks);
BlockInputStreamPtr from = std::make_shared<BlocksBlockInputStream>(std::make_shared<BlocksPtr>(new_mergeable_blocks), mergeable_stream->getHeader());
auto proxy_storage = ProxyStorage::createProxyStorage(global_context.getTable(select_database_name, select_table_name), {from}, QueryProcessingStage::WithMergeableState);
auto proxy_storage = ProxyStorage::createProxyStorage(global_context.getTable(select_table_id), {from}, QueryProcessingStage::WithMergeableState);
InterpreterSelectQuery select(inner_query->clone(), global_context, proxy_storage, SelectQueryOptions(QueryProcessingStage::Complete));
BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
@ -375,7 +331,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr<StorageLiveView> storage, co
if (drop_table)
{
if (storage->global_context.tryGetTable(table_id.database_name, table_id.table_name)) //FIXME
if (storage->global_context.tryGetTable(table_id))
{
try
{
@ -468,9 +424,7 @@ StorageLiveView::~StorageLiveView()
void StorageLiveView::drop(TableStructureWriteLockHolder &)
{
auto table_id = getStorageID();
global_context.removeDependency(
StorageID(select_database_name, select_table_name),
table_id); //FIXME
global_context.removeDependency(select_table_id, table_id);
std::lock_guard lock(mutex);
is_dropped = true;

6
dbms/src/Storages/LiveView/StorageLiveView.h
View File

@ -41,8 +41,7 @@ friend class LiveViewBlockOutputStream;
public:
~StorageLiveView() override;
String getName() const override { return "LiveView"; }
String getSelectDatabaseName() const { return select_database_name; }
String getSelectTableName() const { return select_table_name; }
StorageID getSelectTableID() const { return select_table_id; }
NameAndTypePair getColumn(const String & column_name) const override;
bool hasColumn(const String & column_name) const override;
@ -140,8 +139,7 @@ public:
const Context & context);
private:
String select_database_name;
String select_table_name;
StorageID select_table_id;
ASTPtr inner_query;
Context & global_context;
bool is_temporary = false;

38
dbms/src/Storages/StorageID.h
View File

@ -11,28 +11,36 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_";
struct StorageID
{
String database_name;
String table_name;
String uuid;
StorageID() = delete;
//StorageID() = delete;
StorageID() = default;
//TODO StorageID(const ASTPtr & query_with_one_table, const Context & context) to get db and table names (and maybe uuid) from query
//But there are a lot of different ASTs with db and table name
//And it looks like it depends on https://github.com/ClickHouse/ClickHouse/pull/7774
StorageID(const String & database, const String & table, const String & uuid_ = {})
: database_name(database), table_name(table), uuid(uuid_)
{
assert_not_empty();
}
String getFullTableName() const
{
assert_valid();
return (database_name.empty() ? "" : database_name + ".") + table_name;
}
String getNameForLogs() const
{
return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + " (UUID " + uuid + ")";
assert_valid();
return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + (uuid.empty() ? "" : " (UUID " + uuid + ")");
}
String getId() const
@ -45,13 +53,31 @@ struct StorageID
bool operator<(const StorageID & rhs) const
{
return std::tie(uuid, database_name, table_name) < std::tie(rhs.uuid, rhs.database_name, rhs.table_name);
assert_valid();
/// It's needed for ViewDependencies
if (uuid.empty() && rhs.uuid.empty())
/// If both IDs don't have UUID, compare them like pair of strings
return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name);
else if (!uuid.empty() && !rhs.uuid.empty())
/// If both IDs have UUID, compare UUIDs and ignore database and table name
return uuid < rhs.uuid;
else
/// All IDs without UUID are less, then all IDs with UUID
return uuid.empty();
}
void assert_not_empty() const
bool empty() const
{
if (database_name.empty() && table_name.empty())
return table_name.empty() || (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && uuid.empty());
}
void assert_valid() const
{
if (empty())
throw Exception("empty table name", ErrorCodes::LOGICAL_ERROR);
if (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && uuid.empty() && !database_name.empty())
throw Exception("unexpected database name", ErrorCodes::LOGICAL_ERROR);
}
};

170
dbms/src/Storages/StorageMaterializedView.cpp
View File

@ -27,6 +27,7 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
extern const int INCORRECT_QUERY;
extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW;
extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW;
}
static inline String generateInnerTableName(const String & table_name)
@ -34,39 +35,37 @@ static inline String generateInnerTableName(const String & table_name)
return ".inner." + table_name;
}
static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name)
StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, Context & context, bool is_live_view /*= false*/, bool need_visitor /*= true*/)
{
if (need_visitor)
{
AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr);
visitor.visit(query);
}
auto db_and_table = getDatabaseAndTable(query, 0);
ASTPtr subquery = extractTableExpression(query, 0);
if (!db_and_table && !subquery)
return;
return {}; //FIXME in which cases we cannot get table name?
if (db_and_table)
{
select_table_name = db_and_table->table;
if (db_and_table->database.empty())
{
db_and_table->database = select_database_name;
AddDefaultDatabaseVisitor visitor(select_database_name);
visitor.visit(query);
}
else
select_database_name = db_and_table->database;
//TODO uuid
return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/);
}
else if (auto * ast_select = subquery->as<ASTSelectWithUnionQuery>())
{
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
throw Exception(String("UNION is not supported for ") + (is_live_view ? "LIVE VIEW" : "MATERIALIZED VIEW"),
is_live_view ? ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW : ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
auto & inner_query = ast_select->list_of_selects->children.at(0);
extractDependentTable(inner_query->as<ASTSelectQuery &>(), select_database_name, select_table_name);
return extractDependentTableFromSelectQuery(inner_query->as<ASTSelectQuery &>(), context, is_live_view, false);
}
else
throw Exception("Logical error while creating StorageMaterializedView."
" Could not retrieve table name from select query.",
throw Exception(String("Logical error while creating Storage") + (is_live_view ? "Live" : "Materialized") +
"View. Could not retrieve table name from select query.",
DB::ErrorCodes::LOGICAL_ERROR);
}
@ -105,47 +104,36 @@ StorageMaterializedView::StorageMaterializedView(
if (!query.select)
throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
if (!query.storage && query.to_table.empty())
/// If the destination table is not set, use inner table
has_inner_table = query.to_table_id.empty();
if (has_inner_table && !query.storage)
throw Exception(
"You must specify where to save results of a MaterializedView query: either ENGINE or an existing table in a TO clause",
ErrorCodes::INCORRECT_QUERY);
/// Default value, if only table name exist in the query
select_database_name = local_context.getCurrentDatabase();
if (query.select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
inner_query = query.select->list_of_selects->children.at(0);
auto & select_query = inner_query->as<ASTSelectQuery &>();
extractDependentTable(select_query, select_database_name, select_table_name);
select_table_id = extractDependentTableFromSelectQuery(select_query, local_context);
checkAllowedQueries(select_query);
if (!select_table_name.empty())
global_context.addDependency(
StorageID(select_database_name, select_table_name),
table_id_); //FIXME
// If the destination table is not set, use inner table
if (!query.to_table.empty())
if (!has_inner_table)
target_table_id = query.to_table_id;
else if (attach_)
{
target_database_name = query.to_database;
target_table_name = query.to_table;
/// If there is an ATTACH request, then the internal table must already be created.
//TODO use uuid
target_table_id = StorageID(table_id_.database_name, generateInnerTableName(table_id_.table_name));
}
else
{
target_database_name = table_id_.database_name;
target_table_name = generateInnerTableName(table_id_.table_name);
has_inner_table = true;
}
/// If there is an ATTACH request, then the internal table must already be connected.
if (!attach_ && has_inner_table)
{
/// We will create a query to create an internal table.
auto manual_create_query = std::make_shared<ASTCreateQuery>();
manual_create_query->database = target_database_name;
manual_create_query->table = target_table_name;
manual_create_query->database = table_id_.database_name;
manual_create_query->table = generateInnerTableName(table_id_.table_name);
auto new_columns_list = std::make_shared<ASTColumns>();
new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr());
@ -153,24 +141,15 @@ StorageMaterializedView::StorageMaterializedView(
manual_create_query->set(manual_create_query->columns_list, new_columns_list);
manual_create_query->set(manual_create_query->storage, query.storage->ptr());
/// Execute the query.
try
{
InterpreterCreateQuery create_interpreter(manual_create_query, local_context);
create_interpreter.setInternal(true);
create_interpreter.execute();
}
catch (...)
{
/// In case of any error we should remove dependency to the view.
if (!select_table_name.empty())
global_context.removeDependency(
StorageID(select_database_name, select_table_name),
table_id_); //FIXME
InterpreterCreateQuery create_interpreter(manual_create_query, local_context);
create_interpreter.setInternal(true);
create_interpreter.execute();
throw;
}
target_table_id = global_context.getTable(manual_create_query->database, manual_create_query->table)->getStorageID();
}
if (!select_table_id.empty())
global_context.addDependency(select_table_id, table_id_);
}
NameAndTypePair StorageMaterializedView::getColumn(const String & column_name) const
@ -214,14 +193,14 @@ BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const
}
static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const String & target_database_name, const String & target_table_name)
static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const StorageID & target_table_id)
{
if (global_context.tryGetTable(target_database_name, target_table_name))
if (global_context.tryGetTable(target_table_id))
{
/// We create and execute `drop` query for internal table.
auto drop_query = std::make_shared<ASTDropQuery>();
drop_query->database = target_database_name;
drop_query->table = target_table_name;
drop_query->database = target_table_id.database_name;
drop_query->table = target_table_id.table_name;
drop_query->kind = kind;
ASTPtr ast_drop_query = drop_query;
InterpreterDropQuery drop_interpreter(ast_drop_query, global_context);
@ -233,25 +212,23 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context,
void StorageMaterializedView::drop(TableStructureWriteLockHolder &)
{
auto table_id = getStorageID();
global_context.removeDependency(
StorageID(select_database_name, select_table_name),
table_id); //FIXME
global_context.removeDependency(select_table_id, table_id);
if (has_inner_table && tryGetTargetTable())
executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name);
executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id);
}
void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
if (has_inner_table)
executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name);
executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_table_id);
}
void StorageMaterializedView::checkStatementCanBeForwarded() const
{
if (!has_inner_table)
throw Exception(
"MATERIALIZED VIEW targets existing table " + target_database_name + "." + target_table_name + ". "
"MATERIALIZED VIEW targets existing table " + target_table_id.getNameForLogs() + ". "
+ "Execute the statement directly on it.", ErrorCodes::INCORRECT_QUERY);
}
@ -273,74 +250,59 @@ void StorageMaterializedView::mutate(const MutationCommands & commands, const Co
getTargetTable()->mutate(commands, context);
}
static void executeRenameQuery(Context & global_context, const String & database_name, const String & table_original_name, const String & new_table_name)
{
if (global_context.tryGetTable(database_name, table_original_name))
{
auto rename = std::make_shared<ASTRenameQuery>();
ASTRenameQuery::Table from;
from.database = database_name;
from.table = table_original_name;
ASTRenameQuery::Table to;
to.database = database_name;
to.table = new_table_name;
ASTRenameQuery::Element elem;
elem.from = from;
elem.to = to;
rename->elements.emplace_back(elem);
InterpreterRenameQuery(rename, global_context).execute();
}
}
void StorageMaterializedView::renameInMemory(const String & new_database_name, const String & new_table_name, std::unique_lock<std::mutex> *)
{
//FIXME
if (has_inner_table && tryGetTargetTable())
{
String new_target_table_name = generateInnerTableName(new_table_name);
executeRenameQuery(global_context, target_database_name, target_table_name, new_target_table_name);
target_table_name = new_target_table_name;
auto new_target_table_name = generateInnerTableName(new_table_name);
auto rename = std::make_shared<ASTRenameQuery>();
ASTRenameQuery::Table from;
from.database = target_table_id.database_name;
from.table = target_table_id.table_name;
ASTRenameQuery::Table to;
to.database = target_table_id.database_name;
to.table = new_target_table_name;
ASTRenameQuery::Element elem;
elem.from = from;
elem.to = to;
rename->elements.emplace_back(elem);
InterpreterRenameQuery(rename, global_context).execute();
target_table_id.table_name = new_target_table_name;
}
auto lock = global_context.getLock();
std::unique_lock<std::mutex> name_lock;
auto table_id = getStorageID(&name_lock);
global_context.removeDependencyUnsafe(
StorageID(select_database_name, select_table_name),
table_id);
global_context.removeDependencyUnsafe(select_table_id, table_id);
IStorage::renameInMemory(new_database_name, new_table_name, &name_lock);
global_context.addDependencyUnsafe(
StorageID(select_database_name, select_table_name),
table_id);
auto new_table_id = getStorageID(&name_lock);
global_context.addDependencyUnsafe(select_table_id, new_table_id);
}
void StorageMaterializedView::shutdown()
{
auto table_id = getStorageID();
/// Make sure the dependency is removed after DETACH TABLE
global_context.removeDependency(
StorageID(select_database_name, select_table_name),
table_id); //FIXME
global_context.removeDependency(select_table_id, table_id);
}
StoragePtr StorageMaterializedView::getTargetTable() const
{
return global_context.getTable(target_database_name, target_table_name);
return global_context.getTable(target_table_id);
}
StoragePtr StorageMaterializedView::tryGetTargetTable() const
{
return global_context.tryGetTable(target_database_name, target_table_name);
return global_context.tryGetTable(target_table_id);
}
Strings StorageMaterializedView::getDataPaths() const

9
dbms/src/Storages/StorageMaterializedView.h
View File

@ -9,6 +9,9 @@
namespace DB
{
StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, Context & context, bool is_live_view = false, bool need_visitor = true);
class StorageMaterializedView : public ext::shared_ptr_helper<StorageMaterializedView>, public IStorage
{
friend struct ext::shared_ptr_helper<StorageMaterializedView>;
@ -66,10 +69,8 @@ public:
Strings getDataPaths() const override;
private:
String select_database_name;
String select_table_name;
String target_database_name;
String target_table_name;
StorageID select_table_id;
StorageID target_table_id;
ASTPtr inner_query;
Context & global_context;
bool has_inner_table = false;