mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Improve gathering metadata for backup - part 3.
This commit is contained in:
Vitaly Baranov
parent
461a31f237
commit
44db346fea
@ -467,7 +467,7 @@ void AccessControl::backup(BackupEntriesCollector & backup_entries_collector, Ac
|
||||
void AccessControl::restore(RestorerFromBackup & restorer, const String & data_path_in_backup)
|
||||
{
|
||||
/// The restorer must already know about `data_path_in_backup`, but let's check.
|
||||
restorer.checkPathInBackupToRestoreAccess(data_path_in_backup);
|
||||
restorer.checkPathInBackupIsRegisteredToRestoreAccess(data_path_in_backup);
|
||||
}
|
||||
|
||||
void AccessControl::insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination)
|
||||
|
||||
@ -36,7 +36,6 @@ namespace
|
||||
else
|
||||
return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::string_view BackupEntriesCollector::toString(Stage stage)
|
||||
@ -319,6 +318,9 @@ void BackupEntriesCollector::gatherDatabaseMetadata(
|
||||
|
||||
if (create.getDatabase() != database_name)
|
||||
throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for database {}", backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(database_name));
|
||||
|
||||
String new_database_name = renaming_map.getNewDatabaseName(database_name);
|
||||
database_info.metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql");
|
||||
}
|
||||
|
||||
if (table_name)
|
||||
@ -395,17 +397,19 @@ void BackupEntriesCollector::gatherTablesMetadata()
|
||||
const auto & create = create_table_query->as<const ASTCreateQuery &>();
|
||||
String table_name = create.getTable();
|
||||
|
||||
fs::path data_path_in_backup;
|
||||
if (is_temporary_database)
|
||||
fs::path metadata_path_in_backup, data_path_in_backup;
|
||||
auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name});
|
||||
if (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
{
|
||||
auto table_name_in_backup = renaming_map.getNewTemporaryTableName(table_name);
|
||||
data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup);
|
||||
metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(table_name_in_backup.table) + ".sql");
|
||||
data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name});
|
||||
data_path_in_backup
|
||||
= root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table);
|
||||
metadata_path_in_backup
|
||||
= root_path_in_backup / "metadata" / escapeForFileName(table_name_in_backup.database) / (escapeForFileName(table_name_in_backup.table) + ".sql");
|
||||
data_path_in_backup = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database)
|
||||
/ escapeForFileName(table_name_in_backup.table);
|
||||
}
|
||||
|
||||
/// Add information to `table_infos`.
|
||||
@ -413,6 +417,7 @@ void BackupEntriesCollector::gatherTablesMetadata()
|
||||
res_table_info.database = database;
|
||||
res_table_info.storage = db_table.second;
|
||||
res_table_info.create_table_query = create_table_query;
|
||||
res_table_info.metadata_path_in_backup = metadata_path_in_backup;
|
||||
res_table_info.data_path_in_backup = data_path_in_backup;
|
||||
|
||||
auto partitions_it = database_info.tables.find(table_name);
|
||||
@ -525,9 +530,7 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs()
|
||||
ASTPtr new_create_query = database_info.create_database_query;
|
||||
renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query);
|
||||
|
||||
String new_database_name = renaming_map.getNewDatabaseName(database_name);
|
||||
auto metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql");
|
||||
|
||||
const String & metadata_path_in_backup = database_info.metadata_path_in_backup;
|
||||
backup_entries.emplace_back(metadata_path_in_backup, std::make_shared<BackupEntryFromMemory>(serializeAST(*new_create_query)));
|
||||
}
|
||||
}
|
||||
@ -538,24 +541,11 @@ void BackupEntriesCollector::makeBackupEntriesForTablesDefs()
|
||||
for (const auto & [table_name, table_info] : table_infos)
|
||||
{
|
||||
LOG_TRACE(log, "Adding definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false));
|
||||
bool is_temporary_database = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE);
|
||||
|
||||
ASTPtr new_create_query = table_info.create_table_query;
|
||||
renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query);
|
||||
|
||||
fs::path metadata_path_in_backup;
|
||||
if (is_temporary_database)
|
||||
{
|
||||
auto new_name = renaming_map.getNewTemporaryTableName(table_name.table);
|
||||
metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(new_name) + ".sql");
|
||||
}
|
||||
else
|
||||
{
|
||||
auto new_name = renaming_map.getNewTableName({table_name.database, table_name.table});
|
||||
metadata_path_in_backup
|
||||
= root_path_in_backup / "metadata" / escapeForFileName(new_name.database) / (escapeForFileName(new_name.table) + ".sql");
|
||||
}
|
||||
|
||||
const String & metadata_path_in_backup = table_info.metadata_path_in_backup;
|
||||
backup_entries.emplace_back(metadata_path_in_backup, std::make_shared<BackupEntryFromMemory>(serializeAST(*new_create_query)));
|
||||
}
|
||||
}
|
||||
|
||||
@ -120,6 +120,7 @@ private:
|
||||
{
|
||||
DatabasePtr database;
|
||||
ASTPtr create_database_query;
|
||||
String metadata_path_in_backup;
|
||||
|
||||
struct TableParams
|
||||
{
|
||||
@ -139,6 +140,7 @@ private:
|
||||
StoragePtr storage;
|
||||
TableLockHolder table_lock;
|
||||
ASTPtr create_table_query;
|
||||
String metadata_path_in_backup;
|
||||
std::filesystem::path data_path_in_backup;
|
||||
std::optional<ASTs> partitions;
|
||||
};
|
||||
|
||||
@ -39,7 +39,7 @@ DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & e
|
||||
const String & new_table_name = element.new_table_name;
|
||||
assert(!table_name.empty());
|
||||
assert(!new_table_name.empty());
|
||||
map.setNewTemporaryTableName(table_name, new_table_name);
|
||||
map.setNewTableName({DatabaseCatalog::TEMPORARY_DATABASE, table_name}, {DatabaseCatalog::TEMPORARY_DATABASE, new_table_name});
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@ -39,7 +39,13 @@ namespace ErrorCodes
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr const std::string_view sql_ext = ".sql";
|
||||
String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_char_uppercase)
|
||||
{
|
||||
if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
return fmt::format("{}emporary table {}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(table_name));
|
||||
else
|
||||
return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));
|
||||
}
|
||||
|
||||
String tryGetTableEngine(const IAST & ast)
|
||||
{
|
||||
@ -64,16 +70,6 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
bool RestorerFromBackup::TableKey::operator ==(const TableKey & right) const
|
||||
{
|
||||
return (name == right.name) && (is_temporary == right.is_temporary);
|
||||
}
|
||||
|
||||
bool RestorerFromBackup::TableKey::operator <(const TableKey & right) const
|
||||
{
|
||||
return (name < right.name) || ((name == right.name) && (is_temporary < right.is_temporary));
|
||||
}
|
||||
|
||||
std::string_view RestorerFromBackup::toString(Stage stage)
|
||||
{
|
||||
switch (stage)
|
||||
@ -135,10 +131,11 @@ void RestorerFromBackup::run(bool only_check_access)
|
||||
|
||||
/// Find all the databases and tables which we will read from the backup.
|
||||
setStage(Stage::kFindingTablesInBackup);
|
||||
collectDatabaseAndTableInfos();
|
||||
findDatabasesAndTablesInBackup();
|
||||
|
||||
/// Check access rights.
|
||||
checkAccessForCollectedInfos();
|
||||
checkAccessForObjectsFoundInBackup();
|
||||
|
||||
if (only_check_access)
|
||||
return;
|
||||
|
||||
@ -303,7 +300,7 @@ void RestorerFromBackup::findRootPathsInBackup()
|
||||
", "));
|
||||
}
|
||||
|
||||
void RestorerFromBackup::collectDatabaseAndTableInfos()
|
||||
void RestorerFromBackup::findDatabasesAndTablesInBackup()
|
||||
{
|
||||
database_infos.clear();
|
||||
table_infos.clear();
|
||||
@ -313,22 +310,22 @@ void RestorerFromBackup::collectDatabaseAndTableInfos()
|
||||
{
|
||||
case ASTBackupQuery::ElementType::TABLE:
|
||||
{
|
||||
collectTableInfo({element.database_name, element.table_name}, false, element.partitions);
|
||||
findTableInBackup({element.database_name, element.table_name}, element.partitions);
|
||||
break;
|
||||
}
|
||||
case ASTBackupQuery::ElementType::TEMPORARY_TABLE:
|
||||
{
|
||||
collectTableInfo({element.database_name, element.table_name}, true, element.partitions);
|
||||
findTableInBackup({DatabaseCatalog::TEMPORARY_DATABASE, element.table_name}, element.partitions);
|
||||
break;
|
||||
}
|
||||
case ASTBackupQuery::ElementType::DATABASE:
|
||||
{
|
||||
collectDatabaseInfo(element.database_name, element.except_tables, /* throw_if_no_database_metadata_in_backup= */ true);
|
||||
findDatabaseInBackup(element.database_name, element.except_tables);
|
||||
break;
|
||||
}
|
||||
case ASTBackupQuery::ElementType::ALL:
|
||||
{
|
||||
collectAllDatabasesInfo(element.except_databases, element.except_tables);
|
||||
findEverythingInBackup(element.except_databases, element.except_tables);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -337,9 +334,9 @@ void RestorerFromBackup::collectDatabaseAndTableInfos()
|
||||
LOG_INFO(log, "Will restore {} databases and {} tables", database_infos.size(), table_infos.size());
|
||||
}
|
||||
|
||||
void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional<ASTs> & partitions)
|
||||
void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional<ASTs> & partitions)
|
||||
{
|
||||
String database_name_in_backup = is_temporary_table ? DatabaseCatalog::TEMPORARY_DATABASE : table_name_in_backup.database;
|
||||
bool is_temporary_table = (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE);
|
||||
|
||||
std::optional<fs::path> metadata_path;
|
||||
std::optional<fs::path> root_path_in_use;
|
||||
@ -366,21 +363,20 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_
|
||||
}
|
||||
|
||||
if (!metadata_path)
|
||||
throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Table {} not found in backup", table_name_in_backup.getFullName());
|
||||
throw Exception(
|
||||
ErrorCodes::BACKUP_ENTRY_NOT_FOUND,
|
||||
"{} not found in backup",
|
||||
tableNameWithTypeToString(table_name_in_backup.database, table_name_in_backup.table, true));
|
||||
|
||||
TableKey table_key;
|
||||
fs::path data_path_in_backup;
|
||||
if (is_temporary_table)
|
||||
{
|
||||
data_path_in_backup = *root_path_in_use / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table);
|
||||
table_key.name.table = renaming_map.getNewTemporaryTableName(table_name_in_backup.table);
|
||||
table_key.is_temporary = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
data_path_in_backup
|
||||
= *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table);
|
||||
table_key.name = renaming_map.getNewTableName(table_name_in_backup);
|
||||
}
|
||||
|
||||
auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer();
|
||||
@ -391,25 +387,26 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_
|
||||
ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
||||
renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_table_query);
|
||||
|
||||
if (auto it = table_infos.find(table_key); it != table_infos.end())
|
||||
QualifiedTableName table_name = renaming_map.getNewTableName(table_name_in_backup);
|
||||
|
||||
if (auto it = table_infos.find(table_name); it != table_infos.end())
|
||||
{
|
||||
const TableInfo & table_info = it->second;
|
||||
if (table_info.create_table_query && (serializeAST(*table_info.create_table_query) != serializeAST(*create_table_query)))
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_RESTORE_TABLE,
|
||||
"Extracted two different create queries for the same {}table {}: {} and {}",
|
||||
(is_temporary_table ? "temporary " : ""),
|
||||
table_key.name.getFullName(),
|
||||
"Extracted two different create queries for the same {}: {} and {}",
|
||||
tableNameWithTypeToString(table_name.database, table_name.table, false),
|
||||
serializeAST(*table_info.create_table_query),
|
||||
serializeAST(*create_table_query));
|
||||
}
|
||||
}
|
||||
|
||||
TableInfo & res_table_info = table_infos[table_key];
|
||||
TableInfo & res_table_info = table_infos[table_name];
|
||||
res_table_info.create_table_query = create_table_query;
|
||||
res_table_info.data_path_in_backup = data_path_in_backup;
|
||||
res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_key.name, create_table_query);
|
||||
res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_name, create_table_query);
|
||||
|
||||
if (partitions)
|
||||
{
|
||||
@ -426,27 +423,37 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_
|
||||
}
|
||||
}
|
||||
|
||||
void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_backup, const std::set<DatabaseAndTableName> & except_table_names, bool throw_if_no_database_metadata_in_backup)
|
||||
void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_backup, const std::set<DatabaseAndTableName> & except_table_names)
|
||||
{
|
||||
std::optional<fs::path> metadata_path;
|
||||
std::unordered_set<String> table_names_in_backup;
|
||||
for (const auto & root_path_in_backup : root_paths_in_backup)
|
||||
{
|
||||
fs::path try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql");
|
||||
if (!metadata_path && backup->fileExists(try_metadata_path))
|
||||
fs::path try_metadata_path, try_tables_metadata_path;
|
||||
if (database_name_in_backup == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
{
|
||||
try_tables_metadata_path = root_path_in_backup / "temporary_tables" / "metadata";
|
||||
}
|
||||
else
|
||||
{
|
||||
try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql");
|
||||
try_tables_metadata_path = root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup);
|
||||
}
|
||||
|
||||
if (!metadata_path && !try_metadata_path.empty() && backup->fileExists(try_metadata_path))
|
||||
metadata_path = try_metadata_path;
|
||||
|
||||
Strings file_names = backup->listFiles(root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup));
|
||||
Strings file_names = backup->listFiles(try_tables_metadata_path);
|
||||
for (const String & file_name : file_names)
|
||||
{
|
||||
if (!file_name.ends_with(sql_ext))
|
||||
if (!file_name.ends_with(".sql"))
|
||||
continue;
|
||||
String file_name_without_ext = file_name.substr(0, file_name.length() - sql_ext.length());
|
||||
String file_name_without_ext = file_name.substr(0, file_name.length() - strlen(".sql"));
|
||||
table_names_in_backup.insert(unescapeForFileName(file_name_without_ext));
|
||||
}
|
||||
}
|
||||
|
||||
if (!metadata_path && throw_if_no_database_metadata_in_backup)
|
||||
if (!metadata_path && table_names_in_backup.empty())
|
||||
throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Database {} not found in backup", backQuoteIfNeed(database_name_in_backup));
|
||||
|
||||
if (metadata_path)
|
||||
@ -480,33 +487,26 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac
|
||||
if (except_table_names.contains({database_name_in_backup, table_name_in_backup}))
|
||||
continue;
|
||||
|
||||
collectTableInfo({database_name_in_backup, table_name_in_backup}, /* is_temporary_table= */ false, /* partitions= */ {});
|
||||
findTableInBackup({database_name_in_backup, table_name_in_backup}, /* partitions= */ {});
|
||||
}
|
||||
}
|
||||
|
||||
void RestorerFromBackup::collectAllDatabasesInfo(const std::set<String> & except_database_names, const std::set<DatabaseAndTableName> & except_table_names)
|
||||
void RestorerFromBackup::findEverythingInBackup(const std::set<String> & except_database_names, const std::set<DatabaseAndTableName> & except_table_names)
|
||||
{
|
||||
std::unordered_set<String> database_names_in_backup;
|
||||
std::unordered_set<String> temporary_table_names_in_backup;
|
||||
|
||||
for (const auto & root_path_in_backup : root_paths_in_backup)
|
||||
{
|
||||
Strings file_names = backup->listFiles(root_path_in_backup / "metadata");
|
||||
for (String & file_name : file_names)
|
||||
{
|
||||
if (file_name.ends_with(sql_ext))
|
||||
file_name.resize(file_name.length() - sql_ext.length());
|
||||
if (file_name.ends_with(".sql"))
|
||||
file_name.resize(file_name.length() - strlen(".sql"));
|
||||
database_names_in_backup.emplace(unescapeForFileName(file_name));
|
||||
}
|
||||
|
||||
file_names = backup->listFiles(root_path_in_backup / "temporary_tables" / "metadata");
|
||||
for (String & file_name : file_names)
|
||||
{
|
||||
if (!file_name.ends_with(sql_ext))
|
||||
continue;
|
||||
file_name.resize(file_name.length() - sql_ext.length());
|
||||
temporary_table_names_in_backup.emplace(unescapeForFileName(file_name));
|
||||
}
|
||||
if (backup->hasFiles(root_path_in_backup / "temporary_tables" / "metadata"))
|
||||
database_names_in_backup.emplace(DatabaseCatalog::TEMPORARY_DATABASE);
|
||||
}
|
||||
|
||||
for (const String & database_name_in_backup : database_names_in_backup)
|
||||
@ -514,14 +514,11 @@ void RestorerFromBackup::collectAllDatabasesInfo(const std::set<String> & except
|
||||
if (except_database_names.contains(database_name_in_backup))
|
||||
continue;
|
||||
|
||||
collectDatabaseInfo(database_name_in_backup, except_table_names, /* throw_if_no_database_metadata_in_backup= */ false);
|
||||
findDatabaseInBackup(database_name_in_backup, except_table_names);
|
||||
}
|
||||
|
||||
for (const String & temporary_table_name_in_backup : temporary_table_names_in_backup)
|
||||
collectTableInfo({"", temporary_table_name_in_backup}, /* is_temporary_table= */ true, /* partitions= */ {});
|
||||
}
|
||||
|
||||
void RestorerFromBackup::checkAccessForCollectedInfos() const
|
||||
void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const
|
||||
{
|
||||
AccessRightsElements required_access;
|
||||
for (const auto & database_name : database_infos | boost::adaptors::map_keys)
|
||||
@ -545,7 +542,7 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const
|
||||
if (hasSystemTableEngine(*table_info.create_table_query))
|
||||
continue;
|
||||
|
||||
if (table_name.is_temporary)
|
||||
if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
{
|
||||
if (restore_settings.create_table != RestoreTableCreationMode::kMustExist)
|
||||
required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE);
|
||||
@ -579,7 +576,7 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const
|
||||
flags = AccessType::SHOW_TABLES;
|
||||
}
|
||||
|
||||
required_access.emplace_back(flags, table_name.name.database, table_name.name.table);
|
||||
required_access.emplace_back(flags, table_name.database, table_name.table);
|
||||
}
|
||||
|
||||
if (access_restore_task)
|
||||
@ -611,7 +608,9 @@ void RestorerFromBackup::createDatabases()
|
||||
create_database_query->as<ASTCreateQuery &>().if_not_exists = true;
|
||||
}
|
||||
LOG_TRACE(log, "Creating database {}: {}", backQuoteIfNeed(database_name), serializeAST(*create_database_query));
|
||||
executeCreateQuery(create_database_query);
|
||||
InterpreterCreateQuery interpreter{create_database_query, context};
|
||||
interpreter.setInternal(true);
|
||||
interpreter.execute();
|
||||
}
|
||||
|
||||
DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name);
|
||||
@ -644,15 +643,11 @@ void RestorerFromBackup::createTables()
|
||||
if (tables_to_create.empty())
|
||||
break; /// We've already created all the tables.
|
||||
|
||||
for (const auto & table_key : tables_to_create)
|
||||
for (const auto & table_name : tables_to_create)
|
||||
{
|
||||
auto & table_info = table_infos.at(table_key);
|
||||
auto & table_info = table_infos.at(table_name);
|
||||
|
||||
DatabasePtr database;
|
||||
if (table_key.is_temporary)
|
||||
database = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
|
||||
else
|
||||
database = DatabaseCatalog::instance().getDatabase(table_key.name.database);
|
||||
DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.database);
|
||||
|
||||
bool need_create_table = (restore_settings.create_table != RestoreTableCreationMode::kMustExist);
|
||||
if (need_create_table && hasSystemTableEngine(*table_info.create_table_query))
|
||||
@ -670,9 +665,8 @@ void RestorerFromBackup::createTables()
|
||||
}
|
||||
LOG_TRACE(
|
||||
log,
|
||||
"Creating {}table {}: {}",
|
||||
(table_key.is_temporary ? "temporary " : ""),
|
||||
table_key.name.getFullName(),
|
||||
"Creating {}: {}",
|
||||
tableNameWithTypeToString(table_name.database, table_name.table, false),
|
||||
serializeAST(*create_table_query));
|
||||
|
||||
database->createTableRestoredFromBackup(create_table_query, context, restore_coordination, create_table_timeout_ms);
|
||||
@ -680,9 +674,9 @@ void RestorerFromBackup::createTables()
|
||||
|
||||
table_info.created = true;
|
||||
|
||||
auto resolved_id = table_key.is_temporary
|
||||
? context->resolveStorageID(StorageID{"", table_key.name.table}, Context::ResolveExternal)
|
||||
: context->resolveStorageID(StorageID{table_key.name.database, table_key.name.table}, Context::ResolveGlobal);
|
||||
auto resolved_id = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
? context->resolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal)
|
||||
: context->resolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal);
|
||||
|
||||
auto storage = database->getTable(resolved_id.table_name, context);
|
||||
table_info.storage = storage;
|
||||
@ -697,10 +691,9 @@ void RestorerFromBackup::createTables()
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_RESTORE_TABLE,
|
||||
"The {}table {} has a different definition: {} "
|
||||
"{} has a different definition: {} "
|
||||
"comparing to its definition in the backup: {}",
|
||||
(table_key.is_temporary ? "temporary " : ""),
|
||||
table_key.name.getFullName(),
|
||||
tableNameWithTypeToString(table_name.database, table_name.table, true),
|
||||
serializeAST(*create_table_query),
|
||||
serializeAST(*expected_create_query));
|
||||
}
|
||||
@ -717,9 +710,9 @@ void RestorerFromBackup::createTables()
|
||||
}
|
||||
|
||||
/// Returns the list of tables without dependencies or those which dependencies have been created before.
|
||||
std::vector<RestorerFromBackup::TableKey> RestorerFromBackup::findTablesWithoutDependencies() const
|
||||
std::vector<QualifiedTableName> RestorerFromBackup::findTablesWithoutDependencies() const
|
||||
{
|
||||
std::vector<TableKey> tables_without_dependencies;
|
||||
std::vector<QualifiedTableName> tables_without_dependencies;
|
||||
bool all_tables_created = true;
|
||||
|
||||
for (const auto & [key, table_info] : table_infos)
|
||||
@ -734,7 +727,7 @@ std::vector<RestorerFromBackup::TableKey> RestorerFromBackup::findTablesWithoutD
|
||||
bool all_dependencies_met = true;
|
||||
for (const auto & dependency : table_info.dependencies)
|
||||
{
|
||||
auto it = table_infos.find(TableKey{dependency, false});
|
||||
auto it = table_infos.find(dependency);
|
||||
if ((it != table_infos.end()) && !it->second.created)
|
||||
{
|
||||
all_dependencies_met = false;
|
||||
@ -753,7 +746,7 @@ std::vector<RestorerFromBackup::TableKey> RestorerFromBackup::findTablesWithoutD
|
||||
return {};
|
||||
|
||||
/// Cyclic dependency? We'll try to create those tables anyway but probably it's going to fail.
|
||||
std::vector<TableKey> tables_with_cyclic_dependencies;
|
||||
std::vector<QualifiedTableName> tables_with_cyclic_dependencies;
|
||||
for (const auto & [key, table_info] : table_infos)
|
||||
{
|
||||
if (!table_info.created)
|
||||
@ -766,7 +759,7 @@ std::vector<RestorerFromBackup::TableKey> RestorerFromBackup::findTablesWithoutD
|
||||
"Some tables have cyclic dependency from each other: {}",
|
||||
boost::algorithm::join(
|
||||
tables_with_cyclic_dependencies
|
||||
| boost::adaptors::transformed([](const TableKey & key) -> String { return key.name.getFullName(); }),
|
||||
| boost::adaptors::transformed([](const QualifiedTableName & table_name) -> String { return table_name.getFullName(); }),
|
||||
", "));
|
||||
|
||||
return tables_with_cyclic_dependencies;
|
||||
@ -786,19 +779,12 @@ void RestorerFromBackup::addDataRestoreTasks(DataRestoreTasks && new_tasks)
|
||||
insertAtEnd(data_restore_tasks, std::move(new_tasks));
|
||||
}
|
||||
|
||||
void RestorerFromBackup::checkPathInBackupToRestoreAccess(const String & path)
|
||||
void RestorerFromBackup::checkPathInBackupIsRegisteredToRestoreAccess(const String & path)
|
||||
{
|
||||
if (!access_restore_task || !access_restore_task->hasDataPath(path))
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Path to restore access was not added");
|
||||
}
|
||||
|
||||
void RestorerFromBackup::executeCreateQuery(const ASTPtr & create_query) const
|
||||
{
|
||||
InterpreterCreateQuery interpreter{create_query, context};
|
||||
interpreter.setInternal(true);
|
||||
interpreter.execute();
|
||||
}
|
||||
|
||||
void RestorerFromBackup::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine)
|
||||
{
|
||||
throw Exception(
|
||||
|
||||
@ -48,15 +48,14 @@ public:
|
||||
std::shared_ptr<IRestoreCoordination> getRestoreCoordination() const { return restore_coordination; }
|
||||
std::chrono::seconds getTimeout() const { return timeout; }
|
||||
ContextMutablePtr getContext() const { return context; }
|
||||
void executeCreateQuery(const ASTPtr & create_query) const;
|
||||
|
||||
/// Adds a data restore task which will be later returned by getDataRestoreTasks().
|
||||
/// This function can be called by implementations of IStorage::restoreFromBackup() in inherited storage classes.
|
||||
void addDataRestoreTask(DataRestoreTask && new_task);
|
||||
void addDataRestoreTasks(DataRestoreTasks && new_tasks);
|
||||
|
||||
/// Adds a new data path to restore access control.
|
||||
void checkPathInBackupToRestoreAccess(const String & path);
|
||||
/// Checks that a specified path is already registered to be used for restoring access control.
|
||||
void checkPathInBackupIsRegisteredToRestoreAccess(const String & path);
|
||||
|
||||
/// Reading a backup includes a few stages:
|
||||
enum class Stage
|
||||
@ -104,11 +103,14 @@ private:
|
||||
void run(bool only_check_access);
|
||||
void setStage(Stage new_stage, const String & error_message = {});
|
||||
void findRootPathsInBackup();
|
||||
void collectDatabaseAndTableInfos();
|
||||
void collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional<ASTs> & partitions);
|
||||
void collectDatabaseInfo(const String & database_name_in_backup, const std::set<DatabaseAndTableName> & except_table_names, bool throw_if_no_database_metadata_in_backup);
|
||||
void collectAllDatabasesInfo(const std::set<String> & except_database_names, const std::set<DatabaseAndTableName> & except_table_names);
|
||||
void checkAccessForCollectedInfos() const;
|
||||
|
||||
void findDatabasesAndTablesInBackup();
|
||||
void findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional<ASTs> & partitions);
|
||||
void findDatabaseInBackup(const String & database_name_in_backup, const std::set<DatabaseAndTableName> & except_table_names);
|
||||
void findEverythingInBackup(const std::set<String> & except_database_names, const std::set<DatabaseAndTableName> & except_table_names);
|
||||
|
||||
void checkAccessForObjectsFoundInBackup() const;
|
||||
|
||||
void createDatabases();
|
||||
void createTables();
|
||||
|
||||
@ -128,18 +130,10 @@ private:
|
||||
TableLockHolder table_lock;
|
||||
};
|
||||
|
||||
struct TableKey
|
||||
{
|
||||
QualifiedTableName name;
|
||||
bool is_temporary = false;
|
||||
bool operator ==(const TableKey & right) const;
|
||||
bool operator <(const TableKey & right) const;
|
||||
};
|
||||
|
||||
std::vector<TableKey> findTablesWithoutDependencies() const;
|
||||
std::vector<QualifiedTableName> findTablesWithoutDependencies() const;
|
||||
|
||||
std::unordered_map<String, DatabaseInfo> database_infos;
|
||||
std::map<TableKey, TableInfo> table_infos;
|
||||
std::map<QualifiedTableName, TableInfo> table_infos;
|
||||
std::vector<DataRestoreTask> data_restore_tasks;
|
||||
std::shared_ptr<AccessRestoreTask> access_restore_task;
|
||||
};
|
||||
|
||||
@ -19,7 +19,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int WRONG_DDL_RENAMING_SETTINGS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -31,24 +30,41 @@ namespace
|
||||
{
|
||||
/// CREATE TEMPORARY TABLE
|
||||
String table_name = create.getTable();
|
||||
const auto & new_table_name = data.renaming_map.getNewTemporaryTableName(table_name);
|
||||
if (new_table_name != table_name)
|
||||
create.setTable(new_table_name);
|
||||
QualifiedTableName full_table_name{DatabaseCatalog::TEMPORARY_DATABASE, table_name};
|
||||
const auto & new_table_name = data.renaming_map.getNewTableName(full_table_name);
|
||||
if (new_table_name != full_table_name)
|
||||
{
|
||||
create.setTable(new_table_name.table);
|
||||
if (new_table_name.database != DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
{
|
||||
create.temporary = false;
|
||||
create.setDatabase(new_table_name.database);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
else if (create.table)
|
||||
{
|
||||
/// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW
|
||||
QualifiedTableName qualified_name;
|
||||
qualified_name.table = create.getTable();
|
||||
qualified_name.database = create.getDatabase();
|
||||
QualifiedTableName full_name;
|
||||
full_name.table = create.getTable();
|
||||
full_name.database = create.getDatabase();
|
||||
|
||||
if (!qualified_name.database.empty() && !qualified_name.table.empty())
|
||||
if (!full_name.database.empty() && !full_name.table.empty())
|
||||
{
|
||||
auto new_qualified_name = data.renaming_map.getNewTableName(qualified_name);
|
||||
if (new_qualified_name != qualified_name)
|
||||
auto new_table_name = data.renaming_map.getNewTableName(full_name);
|
||||
if (new_table_name != full_name)
|
||||
{
|
||||
create.setTable(new_qualified_name.table);
|
||||
create.setDatabase(new_qualified_name.database);
|
||||
create.setTable(new_table_name.table);
|
||||
if (new_table_name.database == DatabaseCatalog::TEMPORARY_DATABASE)
|
||||
{
|
||||
create.temporary = true;
|
||||
create.setDatabase("");
|
||||
}
|
||||
else
|
||||
{
|
||||
create.setDatabase(new_table_name.database);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -301,7 +317,7 @@ void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context,
|
||||
void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name)
|
||||
{
|
||||
if (old_table_name.table.empty() || old_table_name.database.empty() || new_table_name.table.empty() || new_table_name.database.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed");
|
||||
throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed");
|
||||
|
||||
auto it = old_to_new_table_names.find(old_table_name);
|
||||
if ((it != old_to_new_table_names.end()))
|
||||
@ -321,7 +337,7 @@ void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name,
|
||||
void DDLRenamingMap::setNewDatabaseName(const String & old_database_name, const String & new_database_name)
|
||||
{
|
||||
if (old_database_name.empty() || new_database_name.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed");
|
||||
throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed");
|
||||
|
||||
auto it = old_to_new_database_names.find(old_database_name);
|
||||
if ((it != old_to_new_database_names.end()))
|
||||
@ -351,28 +367,4 @@ QualifiedTableName DDLRenamingMap::getNewTableName(const QualifiedTableName & ol
|
||||
return {getNewDatabaseName(old_table_name.database), old_table_name.table};
|
||||
}
|
||||
|
||||
void DDLRenamingMap::setNewTemporaryTableName(const String & old_table_name, const String & new_table_name)
|
||||
{
|
||||
if (old_table_name.empty() || new_table_name.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed");
|
||||
|
||||
auto it = old_to_new_temporary_table_names.find(old_table_name);
|
||||
if ((it != old_to_new_temporary_table_names.end()))
|
||||
{
|
||||
if (it->second == new_table_name)
|
||||
return;
|
||||
throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Wrong renaming: it's specified that temporary table {} should be renamed to {} and to {} at the same time",
|
||||
backQuoteIfNeed(old_table_name), backQuoteIfNeed(it->second), backQuoteIfNeed(new_table_name));
|
||||
}
|
||||
old_to_new_temporary_table_names[old_table_name] = new_table_name;
|
||||
}
|
||||
|
||||
const String & DDLRenamingMap::getNewTemporaryTableName(const String & old_table_name) const
|
||||
{
|
||||
auto it = old_to_new_temporary_table_names.find(old_table_name);
|
||||
if (it != old_to_new_temporary_table_names.end())
|
||||
return it->second;
|
||||
return old_table_name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -25,16 +25,13 @@ class DDLRenamingMap
|
||||
public:
|
||||
void setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name);
|
||||
void setNewDatabaseName(const String & old_database_name, const String & new_database_name);
|
||||
void setNewTemporaryTableName(const String & old_table_name, const String & new_table_name);
|
||||
|
||||
QualifiedTableName getNewTableName(const QualifiedTableName & old_table_name) const;
|
||||
const String & getNewDatabaseName(const String & old_database_name) const;
|
||||
const String & getNewTemporaryTableName(const String & old_table_name) const;
|
||||
|
||||
private:
|
||||
std::unordered_map<QualifiedTableName, QualifiedTableName> old_to_new_table_names;
|
||||
std::unordered_map<String, String> old_to_new_database_names;
|
||||
std::unordered_map<String, String> old_to_new_temporary_table_names;
|
||||
};
|
||||
|
||||
/// Visits ASTCreateQuery and changes names of databases or tables.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user