thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 01:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use QualifiedTableName instead of DatabaseAndTableName. Remove mode 'ALL TEMPORARY TABLES'

Browse Source
This commit is contained in:
Vitaly Baranov 2022-06-06 15:52:34 +02:00
parent 21f3bed435
commit cf34883000
12 changed files with 303 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/Backups/BackupEntriesCollector.cpp
View File

@ -163,13 +163,16 @@ void BackupEntriesCollector::collectDatabasesAndTablesInfo()
{
case ASTBackupQuery::ElementType::TABLE:
{
collectTableInfo(element.name, element.partitions, true);
QualifiedTableName table_name{element.database_name, element.table_name};
if (element.is_temporary_database)
table_name.database = DatabaseCatalog::TEMPORARY_DATABASE;
collectTableInfo(table_name, element.partitions, true);
break;
}
case ASTBackupQuery::ElementType::DATABASE:
{
collectDatabaseInfo(element.name.first, element.except_list, true);
collectDatabaseInfo(element.database_name, element.except_list, true);
break;
}
@ -204,7 +207,7 @@ void BackupEntriesCollector::collectDatabasesAndTablesInfo()
}
void BackupEntriesCollector::collectTableInfo(
const DatabaseAndTableName & table_name, const std::optional<ASTs> & partitions, bool throw_if_not_found)
const QualifiedTableName & table_name, const std::optional<ASTs> & partitions, bool throw_if_not_found)
{
/// Gather information about the table.
DatabasePtr database;
@ -215,14 +218,14 @@ void BackupEntriesCollector::collectTableInfo(
if (throw_if_not_found)
{
std::tie(database, storage)
= DatabaseCatalog::instance().getDatabaseAndTable(StorageID{table_name.first, table_name.second}, context);
= DatabaseCatalog::instance().getDatabaseAndTable(StorageID{table_name.database, table_name.table}, context);
table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout);
create_table_query = database->getCreateTableQuery(table_name.second, context);
create_table_query = database->getCreateTableQuery(table_name.table, context);
}
else
{
std::tie(database, storage)
= DatabaseCatalog::instance().tryGetDatabaseAndTable(StorageID{table_name.first, table_name.second}, context);
= DatabaseCatalog::instance().tryGetDatabaseAndTable(StorageID{table_name.database, table_name.table}, context);
if (!storage)
{
consistent &= !table_infos.contains(table_name);
@ -243,7 +246,7 @@ void BackupEntriesCollector::collectTableInfo(
throw;
}
create_table_query = database->tryGetCreateTableQuery(table_name.second, context);
create_table_query = database->tryGetCreateTableQuery(table_name.table, context);
if (!create_table_query)
{
consistent &= !table_infos.contains(table_name);
@ -254,11 +257,18 @@ void BackupEntriesCollector::collectTableInfo(
storage->adjustCreateQueryForBackup(create_table_query);
auto new_table_name = renaming_settings.getNewTableName(table_name);
fs::path data_path_in_backup
= root_path_in_backup / "data" / escapeForFileName(new_table_name.first) / escapeForFileName(new_table_name.second);
= root_path_in_backup / "data" / escapeForFileName(new_table_name.database) / escapeForFileName(new_table_name.table);
/// Check that information is consistent.
const auto & create = create_table_query->as<const ASTCreateQuery &>();
if ((create.getDatabase() != table_name.first) || (create.getTable() != table_name.second))
if (create.getTable() != table_name.table)
{
/// Table was renamed recently.
consistent = false;
return;
}
if ((create.getDatabase() != table_name.database) || (create.temporary && (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE)))
{
/// Table was renamed recently.
consistent = false;
@ -354,7 +364,7 @@ void BackupEntriesCollector::collectDatabaseInfo(const String & database_name, c
if (except_table_names.contains(it->name()))
continue;
collectTableInfo(DatabaseAndTableName{database_name, it->name()}, {}, false);
collectTableInfo(QualifiedTableName{database_name, it->name()}, {}, false);
if (!consistent)
return;
}
@ -381,7 +391,7 @@ void BackupEntriesCollector::checkConsistency()
/// Databases found while we were scanning tables and while we were scanning databases - must be the same.
for (const auto & [table_name, table_info] : table_infos)
{
auto it = database_infos.find(table_name.first);
auto it = database_infos.find(table_name.database);
if (it != database_infos.end())
{
const auto & database_info = it->second;
@ -396,7 +406,7 @@ void BackupEntriesCollector::checkConsistency()
/// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed
/// while we were scanning.
std::set<String> database_names;
std::set<DatabaseAndTableName> table_names;
std::set<QualifiedTableName> table_names;
boost::range::copy(database_infos | boost::adaptors::map_keys, std::inserter(database_names, database_names.end()));
boost::range::copy(table_infos | boost::adaptors::map_keys, std::inserter(table_names, table_names.end()));
@ -431,7 +441,7 @@ void BackupEntriesCollector::makeBackupEntriesForTablesDefs()
{
for (const auto & [table_name, table_info] : table_infos)
{
LOG_TRACE(log, "Adding definition of table {}.{}", backQuoteIfNeed(table_name.first), backQuoteIfNeed(table_name.second));
LOG_TRACE(log, "Adding definition of table {}", table_name.getFullName());
const auto & database = table_info.database;
const auto & storage = table_info.storage;
database->backupCreateTableQuery(*this, storage, table_info.create_table_query);
@ -445,7 +455,7 @@ void BackupEntriesCollector::makeBackupEntriesForTablesData()
for (const auto & [table_name, table_info] : table_infos)
{
LOG_TRACE(log, "Adding data of table {}.{}", backQuoteIfNeed(table_name.first), backQuoteIfNeed(table_name.second));
LOG_TRACE(log, "Adding data of table {}", table_name.getFullName());
const auto & storage = table_info.storage;
const auto & data_path_in_backup = table_info.data_path_in_backup;
const auto & partitions = table_info.partitions;

7
src/Backups/BackupEntriesCollector.h
View File

@ -2,6 +2,7 @@
#include <Backups/BackupSettings.h>
#include <Backups/DDLRenamingVisitor.h>
#include <Core/QualifiedTableName.h>
#include <Parsers/ASTBackupQuery.h>
#include <Storages/IStorage_fwd.h>
#include <Storages/TableLockHolder.h>
@ -83,7 +84,7 @@ private:
void setStage(Stage new_stage, const String & error_message = {});
void calculateRootPathInBackup();
void collectDatabasesAndTablesInfo();
void collectTableInfo(const DatabaseAndTableName & table_name, const std::optional<ASTs> & partitions, bool throw_if_not_found);
void collectTableInfo(const QualifiedTableName & table_name, const std::optional<ASTs> & partitions, bool throw_if_not_found);
void collectDatabaseInfo(const String & database_name, const std::set<String> & except_table_names, bool throw_if_not_found);
void collectAllDatabasesInfo(const std::set<String> & except_database_names);
void checkConsistency();
@ -120,9 +121,9 @@ private:
};
std::unordered_map<String, DatabaseInfo> database_infos;
std::map<DatabaseAndTableName, TableInfo> table_infos;
std::unordered_map<QualifiedTableName, TableInfo> table_infos;
std::optional<std::set<String>> previous_database_names;
std::optional<std::set<DatabaseAndTableName>> previous_table_names;
std::optional<std::set<QualifiedTableName>> previous_table_names;
bool consistent = false;
BackupEntries backup_entries;

20
src/Backups/BackupUtils.cpp
View File

@ -147,22 +147,20 @@ AccessRightsElements getRequiredAccessToBackup(const ASTBackupQuery::Elements &
{
case ASTBackupQuery::TABLE:
{
if (element.is_temp_db)
if (element.is_temporary_database)
break;
AccessFlags flags = AccessType::SHOW_TABLES;
if (!backup_settings.structure_only)
flags |= AccessType::SELECT;
required_access.emplace_back(flags, element.name.first, element.name.second);
required_access.emplace_back(flags, element.database_name, element.table_name);
break;
}
case ASTBackupQuery::DATABASE:
{
if (element.is_temp_db)
break;
AccessFlags flags = AccessType::SHOW_TABLES | AccessType::SHOW_DATABASES;
if (!backup_settings.structure_only)
flags |= AccessType::SELECT;
required_access.emplace_back(flags, element.name.first);
required_access.emplace_back(flags, element.database_name);
/// TODO: It's better to process `element.except_list` somehow.
break;
}
@ -191,7 +189,7 @@ AccessRightsElements getRequiredAccessToRestore(const ASTBackupQuery::Elements &
{
case ASTBackupQuery::TABLE:
{
if (element.is_temp_db)
if (element.is_temporary_database)
{
if (restore_settings.create_table != RestoreTableCreationMode::kMustExist)
required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE);
@ -202,17 +200,11 @@ AccessRightsElements getRequiredAccessToRestore(const ASTBackupQuery::Elements &
flags |= AccessType::CREATE_TABLE;
if (!restore_settings.structure_only)
flags |= AccessType::INSERT;
required_access.emplace_back(flags, element.new_name.first, element.new_name.second);
required_access.emplace_back(flags, element.new_database_name, element.new_table_name);
break;
}
case ASTBackupQuery::DATABASE:
{
if (element.is_temp_db)
{
if (restore_settings.create_table != RestoreTableCreationMode::kMustExist)
required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE);
break;
}
AccessFlags flags = AccessType::SHOW_TABLES | AccessType::SHOW_DATABASES;
if (restore_settings.create_table != RestoreTableCreationMode::kMustExist)
flags |= AccessType::CREATE_TABLE;
@ -220,7 +212,7 @@ AccessRightsElements getRequiredAccessToRestore(const ASTBackupQuery::Elements &
flags |= AccessType::CREATE_DATABASE;
if (!restore_settings.structure_only)
flags |= AccessType::INSERT;
required_access.emplace_back(flags, element.new_name.first);
required_access.emplace_back(flags, element.new_database_name);
break;
}
case ASTBackupQuery::ALL_DATABASES:

4
src/Backups/BackupsWorker.cpp
View File

@ -155,7 +155,7 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
if (in_separate_thread)
query_scope.emplace(cloned_context);
backup_query->setDatabase(cloned_context->getCurrentDatabase());
backup_query->setCurrentDatabase(cloned_context->getCurrentDatabase());
BackupEntries backup_entries;
{
@ -283,7 +283,7 @@ UUID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr conte
if (in_separate_thread)
query_scope.emplace(cloned_context);
restore_query->setDatabase(cloned_context->getCurrentDatabase());
restore_query->setCurrentDatabase(cloned_context->getCurrentDatabase());
BackupFactory::CreateParams backup_open_params;
backup_open_params.open_mode = IBackup::OpenMode::READ;

166
src/Backups/DDLRenamingVisitor.cpp
View File

@ -27,45 +27,57 @@ namespace
{
if (create.table)
{
DatabaseAndTableName table_name;
table_name.second = create.getTable();
/// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE
QualifiedTableName table_name;
table_name.table = create.getTable();
if (create.temporary)
table_name.first = DatabaseCatalog::TEMPORARY_DATABASE;
table_name.database = DatabaseCatalog::TEMPORARY_DATABASE;
else if (create.database)
table_name.first = create.getDatabase();
table_name.database = create.getDatabase();
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Database name specified in the CREATE TABLE query must not be empty");
table_name = data.renaming_settings.getNewTableName(table_name);
auto new_table_name = data.renaming_settings.getNewTableName(table_name);
if (table_name.first == DatabaseCatalog::TEMPORARY_DATABASE)
if (new_table_name != table_name)
{
create.temporary = true;
create.setDatabase("");
create.setTable(new_table_name.table);
if (new_table_name.database == DatabaseCatalog::TEMPORARY_DATABASE)
{
create.temporary = true;
create.setDatabase("");
}
else
{
create.temporary = false;
create.setDatabase(new_table_name.database);
}
}
else
{
create.temporary = false;
create.setDatabase(table_name.first);
}
create.setTable(table_name.second);
}
else if (create.database)
{
/// CREATE DATABASE
String database_name = create.getDatabase();
database_name = data.renaming_settings.getNewDatabaseName(database_name);
create.setDatabase(database_name);
String new_database_name = data.renaming_settings.getNewDatabaseName(database_name);
create.setDatabase(new_database_name);
}
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Database name specified in the CREATE DATABASE query must not be empty");
if (!create.as_table.empty() && !create.as_database.empty())
std::tie(create.as_database, create.as_table) = data.renaming_settings.getNewTableName({create.as_database, create.as_table});
if (!create.to_table_id.table_name.empty() && !create.to_table_id.database_name.empty())
QualifiedTableName as_table{create.as_database, create.as_table};
if (!as_table.table.empty() && !as_table.database.empty())
{
auto to_table = data.renaming_settings.getNewTableName({create.to_table_id.database_name, create.to_table_id.table_name});
create.to_table_id = StorageID{to_table.first, to_table.second};
auto as_table_new = data.renaming_settings.getNewTableName(as_table);
create.as_database = as_table_new.database;
create.as_table = as_table_new.table;
}
QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
if (!to_table.table.empty() && !to_table.database.empty())
{
auto to_table_new = data.renaming_settings.getNewTableName(to_table);
if (to_table_new != to_table)
create.to_table_id = StorageID{to_table_new.database, to_table_new.table};
}
}
@ -83,17 +95,15 @@ namespace
if (!table_id)
return;
const String & db_name = table_id->getDatabaseName();
const String & table_name = table_id->shortName();
if (db_name.empty() || table_name.empty())
QualifiedTableName table_name{table_id->getDatabaseName(), table_id->shortName()};
if (table_name.table.empty() || table_name.database.empty())
return;
String new_db_name, new_table_name;
std::tie(new_db_name, new_table_name) = data.renaming_settings.getNewTableName({db_name, table_name});
if ((new_db_name == db_name) && (new_table_name == table_name))
auto new_table_name = data.renaming_settings.getNewTableName(table_name);
if (new_table_name == table_name)
return;
expr.database_and_table_name = std::make_shared<ASTIdentifier>(Strings{new_db_name, new_table_name});
expr.database_and_table_name = std::make_shared<ASTIdentifier>(new_table_name.getParts());
expr.children.push_back(expr.database_and_table_name);
}
@ -142,44 +152,38 @@ namespace
size_t table_name_index = static_cast<size_t>(-1);
QualifiedTableName qualified_name;
QualifiedTableName table_name;
if (function.name == "Distributed")
qualified_name.table = name;
table_name.table = name;
else
qualified_name = QualifiedTableName::parseFromString(name);
table_name = QualifiedTableName::parseFromString(name);
if (qualified_name.database.empty())
if (table_name.database.empty())
{
std::swap(qualified_name.database, qualified_name.table);
std::swap(table_name.database, table_name.table);
table_name_index = 2;
if (args.size() <= table_name_index)
return;
qualified_name.table = evaluateConstantExpressionForDatabaseName(args[table_name_index], data.context)->as<ASTLiteral &>().value.safeGet<String>();
table_name.table = evaluateConstantExpressionForDatabaseName(args[table_name_index], data.context)->as<ASTLiteral &>().value.safeGet<String>();
}
const String & db_name = qualified_name.database;
const String & table_name = qualified_name.table;
if (db_name.empty() || table_name.empty())
if (table_name.table.empty() || table_name.database.empty())
return;
String new_db_name, new_table_name;
std::tie(new_db_name, new_table_name) = data.renaming_settings.getNewTableName({db_name, table_name});
if ((new_db_name == db_name) && (new_table_name == table_name))
auto new_table_name = data.renaming_settings.getNewTableName(table_name);
if (new_table_name == table_name)
return;
if (table_name_index != static_cast<size_t>(-1))
{
if (new_db_name != db_name)
args[db_name_index] = std::make_shared<ASTLiteral>(new_db_name);
if (new_table_name != table_name)
args[table_name_index] = std::make_shared<ASTLiteral>(new_table_name);
args[db_name_index] = std::make_shared<ASTLiteral>(new_table_name.database);
args[table_name_index] = std::make_shared<ASTLiteral>(new_table_name.table);
}
else
{
args[db_name_index] = std::make_shared<ASTLiteral>(new_db_name);
args.insert(args.begin() + db_name_index + 1, std::make_shared<ASTLiteral>(new_table_name));
args[db_name_index] = std::make_shared<ASTLiteral>(new_table_name.database);
args.insert(args.begin() + db_name_index + 1, std::make_shared<ASTLiteral>(new_table_name.table));
}
}
@ -204,9 +208,9 @@ namespace
return;
auto & elements = dictionary.source->elements->as<ASTExpressionList &>().children;
String db_name, table_name;
size_t db_name_index = static_cast<size_t>(-1);
size_t table_name_index = static_cast<size_t>(-1);
QualifiedTableName table_name;
for (size_t i = 0; i != elements.size(); ++i)
{
@ -215,43 +219,42 @@ namespace
{
if (db_name_index != static_cast<size_t>(-1))
return;
db_name = pair.second->as<ASTLiteral &>().value.safeGet<String>();
table_name.database = pair.second->as<ASTLiteral &>().value.safeGet<String>();
db_name_index = i;
}
else if (pair.first == "table")
{
if (table_name_index != static_cast<size_t>(-1))
return;
table_name = pair.second->as<ASTLiteral &>().value.safeGet<String>();
table_name.table = pair.second->as<ASTLiteral &>().value.safeGet<String>();
table_name_index = i;
}
}
if (db_name.empty() || table_name.empty())
if (table_name.table.empty() || table_name.database.empty())
return;
String new_db_name, new_table_name;
std::tie(new_db_name, new_table_name) = data.renaming_settings.getNewTableName({db_name, table_name});
if ((new_db_name == db_name) && (new_table_name == table_name))
auto new_table_name = data.renaming_settings.getNewTableName(table_name);
if (new_table_name == table_name)
return;
if (new_db_name != db_name)
if (new_table_name.database != table_name.database)
{
auto & pair = elements[db_name_index]->as<ASTPair &>();
pair.replace(pair.second, std::make_shared<ASTLiteral>(new_db_name));
pair.replace(pair.second, std::make_shared<ASTLiteral>(new_table_name.database));
}
if (new_table_name != table_name)
if (new_table_name.table != table_name.table)
{
auto & pair = elements[table_name_index]->as<ASTPair &>();
pair.replace(pair.second, std::make_shared<ASTLiteral>(new_table_name));
pair.replace(pair.second, std::make_shared<ASTLiteral>(new_table_name.table));
}
}
}
void DDLRenamingSettings::setNewTableName(const DatabaseAndTableName & old_table_name, const DatabaseAndTableName & new_table_name)
void DDLRenamingSettings::setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name)
{
if (old_table_name.first.empty() || old_table_name.second.empty() || new_table_name.first.empty() || new_table_name.second.empty())
if (old_table_name.table.empty() || old_table_name.database.empty() || new_table_name.table.empty() || new_table_name.database.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed for DDLRenamingSettings::setNewTableName");
auto it = old_to_new_table_names.find(old_table_name);
@ -259,10 +262,12 @@ void DDLRenamingSettings::setNewTableName(const DatabaseAndTableName & old_table
{
if (it->second == new_table_name)
return;
throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Wrong renaming: it's specified that table {}.{} should be renamed to {}.{} and to {}.{} at the same time",
backQuoteIfNeed(old_table_name.first), backQuoteIfNeed(old_table_name.second),
backQuoteIfNeed(it->second.first), backQuoteIfNeed(it->second.second),
backQuoteIfNeed(new_table_name.first), backQuoteIfNeed(new_table_name.second));
throw Exception(
ErrorCodes::WRONG_DDL_RENAMING_SETTINGS,
"Wrong renaming: it's specified that table {} should be renamed to {} and to {} at the same time",
old_table_name.getFullName(),
it->second.getFullName(),
new_table_name.getFullName());
}
old_to_new_table_names[old_table_name] = new_table_name;
}
@ -301,36 +306,29 @@ void DDLRenamingSettings::setFromBackupQuery(const ASTBackupQuery::Elements & ba
{
case ElementType::TABLE:
{
const String & table_name = element.name.second;
String database_name = element.name.first;
if (element.is_temp_db)
const String & table_name = element.table_name;
const String & new_table_name = element.new_table_name;
String database_name = element.database_name;
String new_database_name = element.new_database_name;
if (element.is_temporary_database)
{
database_name = DatabaseCatalog::TEMPORARY_DATABASE;
new_database_name = DatabaseCatalog::TEMPORARY_DATABASE;
}
assert(!table_name.empty());
assert(!database_name.empty());
const String & new_table_name = element.new_name.second;
String new_database_name = element.new_name.first;
if (element.is_temp_db)
new_database_name = DatabaseCatalog::TEMPORARY_DATABASE;
assert(!new_table_name.empty());
assert(!new_database_name.empty());
setNewTableName({database_name, table_name}, {new_database_name, new_table_name});
break;
}
case ASTBackupQuery::DATABASE:
{
String database_name = element.name.first;
if (element.is_temp_db)
database_name = DatabaseCatalog::TEMPORARY_DATABASE;
const String & database_name = element.database_name;
const String & new_database_name = element.new_database_name;
assert(!database_name.empty());
String new_database_name = element.new_name.first;
if (element.is_temp_db)
new_database_name = DatabaseCatalog::TEMPORARY_DATABASE;
assert(!new_database_name.empty());
setNewDatabaseName(database_name, new_database_name);
break;
}
@ -340,12 +338,12 @@ void DDLRenamingSettings::setFromBackupQuery(const ASTBackupQuery::Elements & ba
}
}
DatabaseAndTableName DDLRenamingSettings::getNewTableName(const DatabaseAndTableName & old_table_name) const
QualifiedTableName DDLRenamingSettings::getNewTableName(const QualifiedTableName & old_table_name) const
{
auto it = old_to_new_table_names.find(old_table_name);
if (it != old_to_new_table_names.end())
return it->second;
return {getNewDatabaseName(old_table_name.first), old_table_name.second};
return {getNewDatabaseName(old_table_name.database), old_table_name.table};
}

9
src/Backups/DDLRenamingVisitor.h
View File

@ -1,16 +1,15 @@
#pragma once
#include <Core/Types.h>
#include <Core/QualifiedTableName.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Parsers/ASTBackupQuery.h>
#include <map>
#include <memory>
#include <unordered_map>
namespace DB
{
using DatabaseAndTableName = std::pair<String, String>;
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
class Context;
@ -23,18 +22,18 @@ class DDLRenamingSettings
public:
DDLRenamingSettings() = default;
void setNewTableName(const DatabaseAndTableName & old_table_name, const DatabaseAndTableName & new_table_name);
void setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name);
void setNewDatabaseName(const String & old_database_name, const String & new_database_name);
void setFromBackupQuery(const ASTBackupQuery & backup_query);
void setFromBackupQuery(const ASTBackupQuery::Elements & backup_query_elements);
/// Changes names according to the renaming.
DatabaseAndTableName getNewTableName(const DatabaseAndTableName & old_table_name) const;
QualifiedTableName getNewTableName(const QualifiedTableName & old_table_name) const;
const String & getNewDatabaseName(const String & old_database_name) const;
private:
std::map<DatabaseAndTableName, DatabaseAndTableName> old_to_new_table_names;
std::unordered_map<QualifiedTableName, QualifiedTableName> old_to_new_table_names;
std::unordered_map<String, String> old_to_new_database_names;
};

45
src/Backups/RestorerFromBackup.cpp
View File

@ -242,12 +242,15 @@ void RestorerFromBackup::collectDatabaseAndTableInfos()
{
case ASTBackupQuery::ElementType::TABLE:
{
collectTableInfo(element.name, element.partitions);
QualifiedTableName table_name{element.database_name, element.table_name};
if (element.is_temporary_database)
table_name.database = DatabaseCatalog::TEMPORARY_DATABASE;
collectTableInfo(table_name, element.partitions);
break;
}
case ASTBackupQuery::ElementType::DATABASE:
{
collectDatabaseInfo(element.name.first, element.except_list);
collectDatabaseInfo(element.database_name, element.except_list);
break;
}
case ASTBackupQuery::ElementType::ALL_DATABASES:
@ -261,14 +264,14 @@ void RestorerFromBackup::collectDatabaseAndTableInfos()
LOG_INFO(log, "Will restore {} databases and {} tables", database_infos.size(), table_infos.size());
}
void RestorerFromBackup::collectTableInfo(const DatabaseAndTableName & table_name_in_backup, const std::optional<ASTs> & partitions)
void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_in_backup, const std::optional<ASTs> & partitions)
{
std::optional<fs::path> metadata_path;
std::optional<fs::path> root_path_in_use;
for (const auto & root_path_in_backup : root_paths_in_backup)
{
fs::path try_metadata_path = root_path_in_backup / "metadata" / escapeForFileName(table_name_in_backup.first)
/ (escapeForFileName(table_name_in_backup.second) + ".sql");
fs::path try_metadata_path = root_path_in_backup / "metadata" / escapeForFileName(table_name_in_backup.database)
/ (escapeForFileName(table_name_in_backup.table) + ".sql");
if (backup->fileExists(try_metadata_path))
{
metadata_path = try_metadata_path;
@ -278,17 +281,11 @@ void RestorerFromBackup::collectTableInfo(const DatabaseAndTableName & table_nam
}
if (!metadata_path)
{
throw Exception(
ErrorCodes::BACKUP_ENTRY_NOT_FOUND,
"Table {}.{} not found in backup",
backQuoteIfNeed(table_name_in_backup.first),
backQuoteIfNeed(table_name_in_backup.second));
}
throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Table {} not found in backup", table_name_in_backup.getFullName());
DatabaseAndTableName table_name = renaming_settings.getNewTableName(table_name_in_backup);
auto table_name = renaming_settings.getNewTableName(table_name_in_backup);
fs::path data_path_in_backup
= *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.first) / escapeForFileName(table_name_in_backup.second);
= *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table);
auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer();
String create_query_str;
@ -305,9 +302,8 @@ void RestorerFromBackup::collectTableInfo(const DatabaseAndTableName & table_nam
{
throw Exception(
ErrorCodes::CANNOT_RESTORE_TABLE,
"Extracted two different create queries for the same table {}.{}: {} and {}",
backQuoteIfNeed(table_name.first),
backQuoteIfNeed(table_name.second),
"Extracted two different create queries for the same table {}: {} and {}",
table_name.getFullName(),
serializeAST(*table_info.create_table_query),
serializeAST(*create_table_query));
}
@ -388,7 +384,7 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac
if (except_table_names.contains(table_name_in_backup))
continue;
collectTableInfo(DatabaseAndTableName{database_name_in_backup, table_name_in_backup}, {});
collectTableInfo(QualifiedTableName{database_name_in_backup, table_name_in_backup}, {});
}
}
@ -459,10 +455,10 @@ void RestorerFromBackup::createTables()
{
for (const auto & [table_name, table_info] : table_infos)
{
DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.first);
DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.database);
if (restore_settings.create_table != RestoreTableCreationMode::kMustExist)
{
LOG_TRACE(log, "Creating table {}.{}", backQuoteIfNeed(table_name.first), backQuoteIfNeed(table_name.second));
LOG_TRACE(log, "Creating table {}", table_name.getFullName());
/// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some
/// database-specific things).
@ -475,12 +471,12 @@ void RestorerFromBackup::createTables()
database->createTableRestoredFromBackup(*this, create_table_query);
}
auto storage = database->getTable(table_name.second, context);
auto storage = database->getTable(table_name.table, context);
table_locks[storage] = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout);
if (!restore_settings.allow_different_table_def)
{
ASTPtr create_table_query = database->getCreateTableQuery(table_name.second, context);
ASTPtr create_table_query = database->getCreateTableQuery(table_name.table, context);
storage->adjustCreateQueryForBackup(create_table_query);
ASTPtr expected_create_query = table_info.create_table_query;
storage->adjustCreateQueryForBackup(expected_create_query);
@ -488,10 +484,9 @@ void RestorerFromBackup::createTables()
{
throw Exception(
ErrorCodes::CANNOT_RESTORE_TABLE,
"The table {}.{} has a different definition: {} "
"The table {} has a different definition: {} "
"comparing to its definition in the backup: {}",
backQuoteIfNeed(table_name.first),
backQuoteIfNeed(table_name.second),
table_name.getFullName(),
serializeAST(*create_table_query),
serializeAST(*expected_create_query));
}

8
src/Backups/RestorerFromBackup.h
View File

@ -95,7 +95,7 @@ private:
void setStage(Stage new_stage, const String & error_message = {});
void findRootPathsInBackup();
void collectDatabaseAndTableInfos();
void collectTableInfo(const DatabaseAndTableName & table_name_in_backup, const std::optional<ASTs> & partitions);
void collectTableInfo(const QualifiedTableName & table_name_in_backup, const std::optional<ASTs> & partitions);
void collectDatabaseInfo(const String & database_name_in_backup, const std::set<String> & except_table_names);
void collectAllDatabasesInfo(const std::set<String> & except_database_names);
void createDatabases();
@ -115,10 +115,10 @@ private:
std::filesystem::path data_path_in_backup;
};
std::map<DatabaseAndTableName, TableInfo> table_infos;
std::unordered_map<QualifiedTableName, TableInfo> table_infos;
std::map<StoragePtr, TableLockHolder> table_locks;
std::map<StoragePtr, std::vector<DataRestoreTask>> data_restore_tasks;
std::unordered_map<StoragePtr, TableLockHolder> table_locks;
std::unordered_map<StoragePtr, std::vector<DataRestoreTask>> data_restore_tasks;
};
}

154
src/Parsers/ASTBackupQuery.cpp
View File

@ -14,66 +14,9 @@ namespace
using Element = ASTBackupQuery::Element;
using ElementType = ASTBackupQuery::ElementType;
void formatType(ElementType type, bool is_temp_db, const IAST::FormatSettings & format)
{
switch (type)
{
case ElementType::TABLE:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "");
if (is_temp_db)
format.ostr << " TEMPORARY TABLE";
else
format.ostr << " TABLE";
format.ostr << (format.hilite ? IAST::hilite_none : "");
break;
}
case ElementType::DATABASE:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "");
if (is_temp_db)
format.ostr << " ALL TEMPORARY TABLES";
else
format.ostr << " DATABASE";
format.ostr << (format.hilite ? IAST::hilite_none : "");
break;
}
case ElementType::ALL_DATABASES:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " ALL DATABASES" << (format.hilite ? IAST::hilite_none : "");
break;
}
}
}
void formatName(const DatabaseAndTableName & name, ElementType type, bool is_temp_db, const IAST::FormatSettings & format)
{
switch (type)
{
case ElementType::TABLE:
{
format.ostr << " ";
if (!is_temp_db && !name.first.empty())
format.ostr << backQuoteIfNeed(name.first) << ".";
format.ostr << backQuoteIfNeed(name.second);
break;
}
case ElementType::DATABASE:
{
if (!is_temp_db)
format.ostr << " " << backQuoteIfNeed(name.first);
break;
}
case ElementType::ALL_DATABASES:
{
break;
}
}
}
void formatPartitions(const ASTs & partitions, const IAST::FormatSettings & format)
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " " << ((partitions.size() == 1) ? "PARTITION" : "PARTITIONS") << " "
format.ostr << " " << (format.hilite ? IAST::hilite_keyword : "") << ((partitions.size() == 1) ? "PARTITION" : "PARTITIONS") << " "
<< (format.hilite ? IAST::hilite_none : "");
bool need_comma = false;
for (const auto & partition : partitions)
@ -85,14 +28,12 @@ namespace
}
}
void formatExceptList(const std::set<String> & except_list, bool show_except_tables, const IAST::FormatSettings & format)
void formatExceptList(const char * except_keyword, const std::set<String> & except_list, const IAST::FormatSettings & format)
{
if (except_list.empty())
return;
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (format.hilite ? IAST::hilite_none : "");
if (show_except_tables)
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "TABLES " << (format.hilite ? IAST::hilite_none : "");
format.ostr << " " << (format.hilite ? IAST::hilite_keyword : "") << except_keyword << (format.hilite ? IAST::hilite_none : "") << " ";
bool need_comma = false;
for (const auto & item : except_list)
@ -105,21 +46,62 @@ namespace
void formatElement(const Element & element, const IAST::FormatSettings & format)
{
formatType(element.type, element.is_temp_db, format);
formatName(element.name, element.type, element.is_temp_db, format);
bool new_name_is_different = (element.new_name != element.name);
if (new_name_is_different)
switch (element.type)
{
format.ostr << " " << (format.hilite ? IAST::hilite_keyword : "") << "AS" << (format.hilite ? IAST::hilite_none : "");
formatName(element.new_name, element.type, element.is_temp_db, format);
case ElementType::TABLE:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "");
if (element.is_temporary_database)
format.ostr << "TEMPORARY TABLE ";
else
format.ostr << "TABLE ";
format.ostr << (format.hilite ? IAST::hilite_none : "");
if (!element.database_name.empty())
format.ostr << backQuoteIfNeed(element.database_name) << ".";
format.ostr << backQuoteIfNeed(element.table_name);
if ((element.new_table_name != element.table_name) || (element.new_database_name != element.database_name))
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : "");
if (!element.new_database_name.empty())
format.ostr << backQuoteIfNeed(element.new_database_name) << ".";
format.ostr << backQuoteIfNeed(element.new_table_name);
}
if (element.partitions)
formatPartitions(*element.partitions, format);
break;
}
case ElementType::DATABASE:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "");
format.ostr << "DATABASE ";
format.ostr << (format.hilite ? IAST::hilite_none : "");
format.ostr << backQuoteIfNeed(element.database_name);
if (element.new_database_name != element.database_name)
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " AS " << (format.hilite ? IAST::hilite_none : "");
format.ostr << backQuoteIfNeed(element.new_database_name);
}
if (!element.except_list.empty())
formatExceptList("EXCEPT TABLES", element.except_list, format);
break;
}
case ElementType::ALL_DATABASES:
{
format.ostr << (format.hilite ? IAST::hilite_keyword : "") << "ALL DATABASES" << (format.hilite ? IAST::hilite_none : "");
if (!element.except_list.empty())
formatExceptList("EXCEPT", element.except_list, format);
break;
}
}
if (element.partitions)
formatPartitions(*element.partitions, format);
bool show_except_tables = ((element.type == ASTBackupQuery::DATABASE) || !element.is_temp_db);
formatExceptList(element.except_list, show_except_tables, format);
}
void formatElements(const std::vector<Element> & elements, const IAST::FormatSettings & format)
@ -128,7 +110,7 @@ namespace
for (const auto & element : elements)
{
if (std::exchange(need_comma, true))
format.ostr << ",";
format.ostr << ", ";
formatElement(element, format);
}
}
@ -191,22 +173,22 @@ namespace
}
void ASTBackupQuery::Element::setDatabase(const String & new_database)
void ASTBackupQuery::Element::setCurrentDatabase(const String & current_database)
{
if ((type == ASTBackupQuery::TABLE) && !is_temp_db)
if ((type == ASTBackupQuery::TABLE) && !is_temporary_database)
{
if (name.first.empty())
name.first = new_database;
if (new_name.first.empty())
new_name.first = new_database;
if (database_name.empty())
database_name = current_database;
if (new_database_name.empty())
new_database_name = current_database;
}
}
void ASTBackupQuery::setDatabase(ASTBackupQuery::Elements & elements, const String & new_database)
void ASTBackupQuery::setCurrentDatabase(ASTBackupQuery::Elements & elements, const String & current_database)
{
for (auto & element : elements)
element.setDatabase(new_database);
element.setCurrentDatabase(current_database);
}
@ -224,7 +206,7 @@ ASTPtr ASTBackupQuery::clone() const
void ASTBackupQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const
{
format.ostr << (format.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? "BACKUP" : "RESTORE")
format.ostr << (format.hilite ? hilite_keyword : "") << ((kind == Kind::BACKUP) ? "BACKUP " : "RESTORE ")
<< (format.hilite ? hilite_none : "");
formatElements(elements, format);
@ -242,7 +224,7 @@ ASTPtr ASTBackupQuery::getRewrittenASTWithoutOnCluster(const WithoutOnClusterAST
auto new_query = std::static_pointer_cast<ASTBackupQuery>(clone());
new_query->cluster.clear();
new_query->settings = rewriteSettingsWithoutOnCluster(new_query->settings, params);
new_query->setDatabase(new_query->elements, params.default_database);
new_query->setCurrentDatabase(new_query->elements, params.default_database);
return new_query;
}

16
src/Parsers/ASTBackupQuery.h
View File

@ -14,7 +14,6 @@ using DatabaseAndTableName = std::pair<String, String>;
* DICTIONARY [db.]dictionary_name [AS [db.]dictionary_name_in_backup] |
* DATABASE database_name [AS database_name_in_backup] [EXCEPT TABLES ...] |
* TEMPORARY TABLE table_name [AS table_name_in_backup] |
* ALL TEMPORARY TABLES [EXCEPT ...] |
* ALL DATABASES [EXCEPT ...] } [,...]
* [ON CLUSTER 'cluster_name']
* TO { File('path/') |
@ -25,7 +24,6 @@ using DatabaseAndTableName = std::pair<String, String>;
* DICTIONARY [db.]dictionary_name_in_backup [AS [db.]dictionary_name] |
* DATABASE database_name_in_backup [AS database_name] [EXCEPT TABLES ...] |
* TEMPORARY TABLE table_name_in_backup [AS table_name] |
* ALL TEMPORARY TABLES [EXCEPT ...] |
* ALL DATABASES [EXCEPT ...] } [,...]
* [ON CLUSTER 'cluster_name']
* FROM {File(...) | Disk(...)}
@ -59,18 +57,20 @@ public:
struct Element
{
ElementType type;
DatabaseAndTableName name;
DatabaseAndTableName new_name;
bool is_temp_db = false;
String table_name;
String database_name;
bool is_temporary_database = false;
String new_table_name; /// usually the same as `table_name`, can be different in case of using AS <new_name>
String new_database_name; /// usually the same as `database_name`, can be different in case of using AS <new_name>
std::optional<ASTs> partitions;
std::set<String> except_list;
void setDatabase(const String & new_database);
void setCurrentDatabase(const String & current_database);
};
using Elements = std::vector<Element>;
static void setDatabase(Elements & elements, const String & new_database);
void setDatabase(const String & new_database) { setDatabase(elements, new_database); }
static void setCurrentDatabase(Elements & elements, const String & current_database);
void setCurrentDatabase(const String & current_database) { setCurrentDatabase(elements, current_database); }
Elements elements;

197
src/Parsers/ParserBackupQuery.cpp
View File

@ -21,74 +21,6 @@ namespace
using Element = ASTBackupQuery::Element;
using ElementType = ASTBackupQuery::ElementType;
bool parseType(IParser::Pos & pos, Expected & expected, ElementType & type, bool & is_temp_db)
{
is_temp_db = false;
if (ParserKeyword{"TABLE"}.ignore(pos, expected) || ParserKeyword{"DICTIONARY"}.ignore(pos, expected))
{
type = ElementType::TABLE;
return true;
}
if (ParserKeyword{"TEMPORARY TABLE"}.ignore(pos, expected))
{
type = ElementType::TABLE;
is_temp_db = true;
return true;
}
if (ParserKeyword{"DATABASE"}.ignore(pos, expected))
{
type = ElementType::DATABASE;
return true;
}
if (ParserKeyword{"ALL TEMPORARY TABLES"}.ignore(pos, expected))
{
type = ElementType::DATABASE;
is_temp_db = true;
return true;
}
if (ParserKeyword{"ALL DATABASES"}.ignore(pos, expected))
{
type = ElementType::ALL_DATABASES;
return true;
}
return false;
}
bool parseName(IParser::Pos & pos, Expected & expected, ElementType type, bool is_temp_db, DatabaseAndTableName & name)
{
name.first.clear();
name.second.clear();
switch (type)
{
case ElementType::TABLE:
{
if (is_temp_db)
{
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
name.second = getIdentifierName(ast);
return true;
}
return parseDatabaseAndTableName(pos, expected, name.first, name.second);
}
case ElementType::DATABASE:
{
if (is_temp_db)
return false;
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
name.first = getIdentifierName(ast);
return true;
}
default:
return false;
}
}
bool parsePartitions(IParser::Pos & pos, Expected & expected, std::optional<ASTs> & partitions)
{
if (!ParserKeyword{"PARTITION"}.ignore(pos, expected) && !ParserKeyword{"PARTITIONS"}.ignore(pos, expected))
@ -110,71 +42,104 @@ namespace
return true;
}
bool parseExceptList(IParser::Pos & pos, Expected & expected, bool parse_except_tables, std::set<String> & except_list)
{
if (!ParserKeyword{parse_except_tables ? "EXCEPT TABLES" : "EXCEPT"}.ignore(pos, expected))
return false;
std::set<String> result;
auto parse_list_element = [&]
{
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
result.insert(getIdentifierName(ast));
return true;
};
if (!ParserList::parseUtil(pos, expected, parse_list_element, false))
return false;
except_list = std::move(result);
return true;
}
bool parseElement(IParser::Pos & pos, Expected & expected, Element & entry)
bool parseExceptList(IParser::Pos & pos, Expected & expected, const char * except_keyword, std::set<String> & except_list)
{
return IParserBase::wrapParseImpl(pos, [&]
{
ElementType type;
bool is_temp_db = false;
if (!parseType(pos, expected, type, is_temp_db))
if (!ParserKeyword{except_keyword}.ignore(pos, expected))
return false;
DatabaseAndTableName name;
if ((type == ElementType::TABLE) || (type == ElementType::DATABASE && !is_temp_db))
std::set<String> result;
auto parse_list_element = [&]
{
if (!parseName(pos, expected, type, is_temp_db, name))
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
}
result.insert(getIdentifierName(ast));
return true;
};
if (!ParserList::parseUtil(pos, expected, parse_list_element, false))
return false;
DatabaseAndTableName new_name = name;
if (ParserKeyword{"AS"}.ignore(pos, expected))
except_list = std::move(result);
return true;
});
}
bool parseElement(IParser::Pos & pos, Expected & expected, Element & element)
{
return IParserBase::wrapParseImpl(pos, [&]
{
if (ParserKeyword{"TABLE"}.ignore(pos, expected))
{
if ((type == ElementType::TABLE) || (type == ElementType::DATABASE && !is_temp_db))
element.type = ElementType::TABLE;
if (!parseDatabaseAndTableName(pos, expected, element.database_name, element.table_name))
return false;
element.new_database_name = element.database_name;
element.new_table_name = element.table_name;
if (ParserKeyword("AS").ignore(pos, expected))
{
if (!parseName(pos, expected, type, is_temp_db, new_name))
if (!parseDatabaseAndTableName(pos, expected, element.new_database_name, element.new_table_name))
return false;
}
parsePartitions(pos, expected, element.partitions);
return true;
}
std::optional<ASTs> partitions;
if (type == ElementType::TABLE)
parsePartitions(pos, expected, partitions);
std::set<String> except_list;
if ((type == ElementType::DATABASE) || (type == ElementType::ALL_DATABASES))
if (ParserKeyword{"TEMPORARY TABLE"}.ignore(pos, expected))
{
bool parse_except_tables = ((type == ElementType::DATABASE) && !is_temp_db);
parseExceptList(pos, expected, parse_except_tables, except_list);
element.type = ElementType::TABLE;
element.is_temporary_database = true;
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
element.table_name = getIdentifierName(ast);
element.new_table_name = element.table_name;
if (ParserKeyword("AS").ignore(pos, expected))
{
ast = nullptr;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
element.new_table_name = getIdentifierName(ast);
}
return true;
}
entry.type = type;
entry.name = std::move(name);
entry.new_name = std::move(new_name);
entry.is_temp_db = is_temp_db;
entry.partitions = std::move(partitions);
entry.except_list = std::move(except_list);
return true;
if (ParserKeyword{"DATABASE"}.ignore(pos, expected))
{
element.type = ElementType::DATABASE;
ASTPtr ast;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
element.database_name = getIdentifierName(ast);
element.new_database_name = element.database_name;
if (ParserKeyword("AS").ignore(pos, expected))
{
ast = nullptr;
if (!ParserIdentifier{}.parse(pos, ast, expected))
return false;
element.new_database_name = getIdentifierName(ast);
}
parseExceptList(pos, expected, "EXCEPT TABLES", element.except_list);
return true;
}
if (ParserKeyword{"ALL DATABASES"}.ignore(pos, expected))
{
element.type = ElementType::ALL_DATABASES;
parseExceptList(pos, expected, "EXCEPT", element.except_list);
return true;
}
return false;
});
}

2
src/Parsers/ParserBackupQuery.h
View File

@ -10,7 +10,6 @@ namespace DB
* DICTIONARY [db.]dictionary_name [AS [db.]dictionary_name_in_backup] |
* DATABASE database_name [AS database_name_in_backup] [EXCEPT TABLES ...] |
* TEMPORARY TABLE table_name [AS table_name_in_backup] |
* ALL TEMPORARY TABLES [EXCEPT ...] |
* ALL DATABASES [EXCEPT ...] } [,...]
* [ON CLUSTER 'cluster_name']
* TO { File('path/') |
@ -21,7 +20,6 @@ namespace DB
* DICTIONARY [db.]dictionary_name_in_backup [AS [db.]dictionary_name] |
* DATABASE database_name_in_backup [AS database_name] [EXCEPT TABLES ...] |
* TEMPORARY TABLE table_name_in_backup [AS table_name] |
* ALL TEMPORARY TABLES [EXCEPT ...] |
* ALL DATABASES [EXCEPT ...] } [,...]
* [ON CLUSTER 'cluster_name']
* FROM {File(...) | Disk(...)}