thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #70100 from ClickHouse/backport/24.9/69563

Browse Source 
Backport #69563 to 24.9: Improve restoring of access entities' dependencies #2
This commit is contained in:
robot-ch-test-poll3 2024-09-29 01:25:04 +02:00 committed by GitHub
commit a08a8ee516
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
60 changed files with 1232 additions and 501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

757
src/Access/AccessBackup.cpp
View File

@ -30,6 +30,7 @@ namespace ErrorCodes
{
extern const int CANNOT_RESTORE_TABLE;
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
extern const int ACCESS_ENTITY_NOT_FOUND;
extern const int LOGICAL_ERROR;
}
@ -41,6 +42,7 @@ namespace
{
std::unordered_map<UUID, AccessEntityPtr> entities;
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
std::unordered_map<UUID, AccessEntityPtr> dependents;
BackupEntryPtr toBackupEntry() const
{
@ -72,6 +74,24 @@ namespace
}
}
if (!dependents.empty())
{
if (!dependencies.empty())
writeText("\n", buf);
writeText("DEPENDENTS\n", buf);
for (const auto & [id, entity] : dependents)
{
writeText(id, buf);
writeChar('\t', buf);
writeText(entity->getTypeInfo().name, buf);
writeChar('\t', buf);
writeText(entity->getName(), buf);
writeChar('\n', buf);
writeText(serializeAccessEntity(*entity), buf);
writeChar('\n', buf);
}
}
return std::make_shared<BackupEntryFromMemory>(buf.str());
}
@ -81,59 +101,71 @@ namespace
{
AccessEntitiesInBackup res;
bool dependencies_found = false;
bool reading_dependencies = false;
bool reading_dependents = false;
while (!buf->eof())
{
String line;
readStringUntilNewlineInto(line, *buf);
buf->ignore();
if (line == "DEPENDENCIES")
{
dependencies_found = true;
break;
reading_dependencies = true;
reading_dependents = false;
continue;
}
else if (line == "DEPENDENTS")
{
reading_dependents = true;
reading_dependencies = false;
continue;
}
else if (line.empty())
{
continue;
}
UUID id = parse<UUID>(line.substr(0, line.find('\t')));
line.clear();
size_t separator1 = line.find('\t');
size_t separator2 = line.find('\t', separator1 + 1);
if ((separator1 == String::npos) || (separator2 == String::npos))
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Separators not found in line {}", line);
String queries;
while (!buf->eof())
UUID id = parse<UUID>(line.substr(0, separator1));
AccessEntityType type = AccessEntityTypeInfo::parseType(line.substr(separator1 + 1, separator2 - separator1 - 1));
String name = line.substr(separator2 + 1);
if (reading_dependencies)
{
String query;
readStringUntilNewlineInto(query, *buf);
buf->ignore();
if (query.empty())
break;
if (!queries.empty())
queries.append("\n");
queries.append(query);
res.dependencies.emplace(id, std::pair{name, type});
}
AccessEntityPtr entity = deserializeAccessEntity(queries);
res.entities.emplace(id, entity);
}
if (dependencies_found)
{
while (!buf->eof())
else
{
String id_as_string;
readStringInto(id_as_string, *buf);
buf->ignore();
UUID id = parse<UUID>(id_as_string);
String queries;
while (!buf->eof())
{
String query;
readStringUntilNewlineInto(query, *buf);
buf->ignore();
if (query.empty())
break;
if (!queries.empty())
queries.append("\n");
queries.append(query);
}
String type_as_string;
readStringInto(type_as_string, *buf);
buf->ignore();
AccessEntityType type = AccessEntityTypeInfo::parseType(type_as_string);
AccessEntityPtr entity = deserializeAccessEntity(queries);
String name;
readStringInto(name, *buf);
buf->ignore();
if (name != entity->getName())
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected name {} is specified for {}", name, entity->formatTypeWithName());
if (type != entity->getType())
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected type {} is specified for {}", AccessEntityTypeInfo::get(type).name, entity->formatTypeWithName());
if (!res.entities.contains(id))
res.dependencies.emplace(id, std::pair{name, type});
if (reading_dependents)
res.dependents.emplace(id, entity);
else
res.entities.emplace(id, entity);
}
}
@ -146,190 +178,59 @@ namespace
}
}
};
std::vector<UUID> findDependencies(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
{
std::vector<UUID> res;
for (const auto & entity : entities | boost::adaptors::map_values)
insertAtEnd(res, entity->findDependencies());
/// Remove duplicates in the list of dependencies (some entities can refer to other entities).
::sort(res.begin(), res.end());
res.erase(std::unique(res.begin(), res.end()), res.end());
for (const auto & id : entities | boost::adaptors::map_keys)
{
auto it = std::lower_bound(res.begin(), res.end(), id);
if ((it != res.end()) && (*it == id))
res.erase(it);
}
return res;
}
std::unordered_map<UUID, std::pair<String, AccessEntityType>> readDependenciesNamesAndTypes(const std::vector<UUID> & dependencies, const AccessControl & access_control)
{
std::unordered_map<UUID, std::pair<String, AccessEntityType>> res;
for (const auto & id : dependencies)
{
if (auto name_and_type = access_control.tryReadNameWithType(id))
res.emplace(id, name_and_type.value());
}
return res;
}
/// Checks if new entities (which we're going to restore) already exist,
/// and either skips them or throws an exception depending on the restore settings.
void checkExistingEntities(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
std::unordered_map<UUID, UUID> & old_to_new_id,
const AccessControl & access_control,
RestoreAccessCreationMode creation_mode)
{
if (creation_mode == RestoreAccessCreationMode::kReplace)
return;
auto should_skip = [&](const std::pair<UUID, AccessEntityPtr> & id_and_entity)
{
const auto & id = id_and_entity.first;
const auto & entity = *id_and_entity.second;
auto existing_id = access_control.find(entity.getType(), entity.getName());
if (!existing_id)
{
return false;
}
else if (creation_mode == RestoreAccessCreationMode::kCreateIfNotExists)
{
old_to_new_id[id] = *existing_id;
return true;
}
else
{
throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", entity.formatTypeWithName());
}
};
std::erase_if(entities, should_skip);
}
/// If new entities (which we're going to restore) depend on other entities which are not going to be restored or not present in the backup
/// then we should try to replace those dependencies with already existing entities.
void resolveDependencies(const std::unordered_map<UUID, std::pair<String, AccessEntityType>> & dependencies,
std::unordered_map<UUID, UUID> & old_to_new_ids,
const AccessControl & access_control,
bool allow_unresolved_dependencies)
{
for (const auto & [id, name_and_type] : dependencies)
{
std::optional<UUID> new_id;
if (allow_unresolved_dependencies)
new_id = access_control.find(name_and_type.second, name_and_type.first);
else
new_id = access_control.getID(name_and_type.second, name_and_type.first);
if (new_id)
old_to_new_ids.emplace(id, *new_id);
}
}
/// Generates random IDs for the new entities.
void generateRandomIDs(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, std::unordered_map<UUID, UUID> & old_to_new_ids)
{
Poco::UUIDGenerator generator;
for (auto & [id, entity] : entities)
{
UUID new_id;
generator.createRandom().copyTo(reinterpret_cast<char *>(&new_id));
old_to_new_ids.emplace(id, new_id);
id = new_id;
}
}
/// Updates dependencies of the new entities using a specified map.
void replaceDependencies(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
for (auto & entity : entities | boost::adaptors::map_values)
IAccessEntity::replaceDependencies(entity, old_to_new_ids);
}
AccessRightsElements getRequiredAccessToRestore(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
{
AccessRightsElements res;
for (const auto & entity : entities | boost::adaptors::map_values)
{
auto entity_type = entity->getType();
switch (entity_type)
{
case User::TYPE:
{
const auto & user = typeid_cast<const User &>(*entity);
res.emplace_back(AccessType::CREATE_USER);
auto elements = user.access.getElements();
for (auto & element : elements)
{
if (element.is_partial_revoke)
continue;
element.grant_option = true;
res.emplace_back(element);
}
if (!user.granted_roles.isEmpty())
res.emplace_back(AccessType::ROLE_ADMIN);
break;
}
case Role::TYPE:
{
const auto & role = typeid_cast<const Role &>(*entity);
res.emplace_back(AccessType::CREATE_ROLE);
auto elements = role.access.getElements();
for (auto & element : elements)
{
if (element.is_partial_revoke)
continue;
element.grant_option = true;
res.emplace_back(element);
}
if (!role.granted_roles.isEmpty())
res.emplace_back(AccessType::ROLE_ADMIN);
break;
}
case SettingsProfile::TYPE:
{
res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
break;
}
case RowPolicy::TYPE:
{
const auto & policy = typeid_cast<const RowPolicy &>(*entity);
res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName());
break;
}
case Quota::TYPE:
{
res.emplace_back(AccessType::CREATE_QUOTA);
break;
}
default:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type));
}
}
return res;
}
}
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
const String & data_path_in_backup,
size_t counter,
const AccessControl & access_control)
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
const std::vector<UUID> & entities_ids,
const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
bool write_dependents,
const String & data_path_in_backup)
{
auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control);
AccessEntitiesInBackup ab;
boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end()));
ab.dependencies = std::move(dependencies);
String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ...
std::unordered_set<UUID> entities_ids_set;
for (const auto & id : entities_ids)
entities_ids_set.emplace(id);
for (const auto & id : entities_ids)
{
auto it = all_entities.find(id);
if (it != all_entities.end())
{
AccessEntityPtr entity = it->second;
ab.entities.emplace(id, entity);
auto dependencies = entity->findDependencies();
for (const auto & dependency_id : dependencies)
{
if (!entities_ids_set.contains(dependency_id))
{
auto it_dependency = all_entities.find(dependency_id);
if (it_dependency != all_entities.end())
{
auto dependency_entity = it_dependency->second;
ab.dependencies.emplace(dependency_id, std::make_pair(dependency_entity->getName(), dependency_entity->getType()));
}
}
}
}
}
if (write_dependents)
{
for (const auto & [id, possible_dependent] : all_entities)
{
if (!entities_ids_set.contains(id) && possible_dependent->hasDependencies(entities_ids_set))
{
auto dependent = possible_dependent->clone();
dependent->clearAllExceptDependencies();
ab.dependents.emplace(id, dependent);
}
}
}
String filename = fmt::format("access-{}.txt", UUIDHelpers::generateV4());
String file_path_in_backup = fs::path{data_path_in_backup} / filename;
return {file_path_in_backup, ab.toBackupEntry()};
}
@ -339,61 +240,411 @@ AccessRestorerFromBackup::AccessRestorerFromBackup(
const BackupPtr & backup_, const RestoreSettings & restore_settings_)
: backup(backup_)
, creation_mode(restore_settings_.create_access)
, allow_unresolved_dependencies(restore_settings_.allow_unresolved_access_dependencies)
, skip_unresolved_dependencies(restore_settings_.skip_unresolved_access_dependencies)
, update_dependents(restore_settings_.update_access_entities_dependents)
, log(getLogger("AccessRestorerFromBackup"))
{
}
AccessRestorerFromBackup::~AccessRestorerFromBackup() = default;
void AccessRestorerFromBackup::addDataPath(const String & data_path)
void AccessRestorerFromBackup::addDataPath(const String & data_path_in_backup)
{
if (!data_paths.emplace(data_path).second)
return;
if (loaded)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities already loaded");
fs::path data_path_in_backup_fs = data_path;
Strings filenames = backup->listFiles(data_path, /*recursive*/ false);
if (filenames.empty())
return;
for (const String & filename : filenames)
{
if (!filename.starts_with("access") || !filename.ends_with(".txt"))
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"",
String{data_path_in_backup_fs / filename});
}
::sort(filenames.begin(), filenames.end());
for (const String & filename : filenames)
{
String filepath_in_backup = data_path_in_backup_fs / filename;
auto read_buffer_from_backup = backup->readFile(filepath_in_backup);
auto ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup);
boost::range::copy(ab.entities, std::back_inserter(entities));
boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end()));
}
for (const auto & id : entities | boost::adaptors::map_keys)
dependencies.erase(id);
if (std::find(data_paths_in_backup.begin(), data_paths_in_backup.end(), data_path_in_backup) == data_paths_in_backup.end())
data_paths_in_backup.emplace_back(data_path_in_backup);
}
void AccessRestorerFromBackup::loadFromBackup()
{
if (loaded)
return;
/// Parse files "access*.txt" found in the added data paths in the backup.
for (size_t data_path_index = 0; data_path_index != data_paths_in_backup.size(); ++data_path_index)
{
const String & data_path_in_backup = data_paths_in_backup[data_path_index];
fs::path data_path_in_backup_fs = data_path_in_backup;
Strings filenames = backup->listFiles(data_path_in_backup_fs, /*recursive*/ false);
if (filenames.empty())
continue;
for (const String & filename : filenames)
{
if (!filename.starts_with("access") || !filename.ends_with(".txt"))
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"",
String{data_path_in_backup_fs / filename});
}
for (const String & filename : filenames)
{
String filepath_in_backup = data_path_in_backup_fs / filename;
AccessEntitiesInBackup ab;
try
{
auto read_buffer_from_backup = backup->readFile(filepath_in_backup);
ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup);
}
catch (Exception & e)
{
e.addMessage("While reading access entities from {} in backup", filepath_in_backup);
throw;
}
for (const auto & [id, entity] : ab.entities)
{
auto it = entity_infos.find(id);
if (it == entity_infos.end())
{
it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first;
}
EntityInfo & entity_info = it->second;
entity_info.entity = entity;
entity_info.restore = true;
entity_info.data_path_index = data_path_index;
}
for (const auto & [id, name_and_type] : ab.dependencies)
{
auto it = entity_infos.find(id);
if (it == entity_infos.end())
{
it = entity_infos.emplace(id, EntityInfo{.id = id, .name = name_and_type.first, .type = name_and_type.second}).first;
}
EntityInfo & entity_info = it->second;
entity_info.is_dependency = true;
}
for (const auto & [id, entity] : ab.dependents)
{
auto it = entity_infos.find(id);
if (it == entity_infos.end())
{
it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first;
}
EntityInfo & entity_info = it->second;
if (!entity_info.restore)
entity_info.entity = entity;
}
}
}
loaded = true;
}
AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const
{
return getRequiredAccessToRestore(entities);
if (!loaded)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded");
AccessRightsElements res;
for (const auto & [id, entity_info] : entity_infos)
{
if (!entity_info.restore)
continue;
const auto & entity = entity_info.entity;
auto entity_type = entity->getType();
switch (entity_type)
{
case User::TYPE:
{
const auto & user = typeid_cast<const User &>(*entity);
res.emplace_back(AccessType::CREATE_USER);
auto elements = user.access.getElements();
for (auto & element : elements)
{
if (element.is_partial_revoke)
continue;
element.grant_option = true;
res.emplace_back(element);
}
if (!user.granted_roles.isEmpty())
res.emplace_back(AccessType::ROLE_ADMIN);
break;
}
case Role::TYPE:
{
const auto & role = typeid_cast<const Role &>(*entity);
res.emplace_back(AccessType::CREATE_ROLE);
auto elements = role.access.getElements();
for (auto & element : elements)
{
if (element.is_partial_revoke)
continue;
element.grant_option = true;
res.emplace_back(element);
}
if (!role.granted_roles.isEmpty())
res.emplace_back(AccessType::ROLE_ADMIN);
break;
}
case SettingsProfile::TYPE:
{
res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
break;
}
case RowPolicy::TYPE:
{
const auto & policy = typeid_cast<const RowPolicy &>(*entity);
res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName());
break;
}
case Quota::TYPE:
{
res.emplace_back(AccessType::CREATE_QUOTA);
break;
}
default:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type));
}
}
return res;
}
std::vector<std::pair<UUID, AccessEntityPtr>> AccessRestorerFromBackup::getAccessEntities(const AccessControl & access_control) const
void AccessRestorerFromBackup::generateRandomIDsAndResolveDependencies(const AccessControl & access_control)
{
auto new_entities = entities;
if (ids_assigned)
return;
if (!loaded)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded");
/// Calculate `new_id` for each entity info.
/// Check which ones of the loaded access entities already exist.
/// Generate random UUIDs for access entities which we're going to restore if they don't exist.
for (auto & [id, entity_info] : entity_infos)
{
const String & name = entity_info.name;
auto type = entity_info.type;
if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kReplace))
{
entity_info.new_id = UUIDHelpers::generateV4();
LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id);
continue;
}
if (auto existing_id = access_control.find(type, name))
{
if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kCreate))
{
throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists",
AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
}
bool was_going_to_restore = entity_info.restore;
entity_info.new_id = *existing_id;
entity_info.restore = false;
LOG_TRACE(log, "{}: Found with UUID {}{}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *existing_id,
(was_going_to_restore ? ", will not restore" : ""));
}
else
{
if (entity_info.restore)
{
entity_info.new_id = UUIDHelpers::generateV4();
LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id);
}
else if (skip_unresolved_dependencies)
{
LOG_TRACE(log, "{}: Not found, ignoring", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
}
else
{
throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "Cannot resolve {} while restoring from backup",
AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
}
}
}
/// Prepare map from old UUIDs to new UUIDs.
std::unordered_set<UUID> ids_to_restore;
std::unordered_map<UUID, UUID> old_to_new_ids;
checkExistingEntities(new_entities, old_to_new_ids, access_control, creation_mode);
resolveDependencies(dependencies, old_to_new_ids, access_control, allow_unresolved_dependencies);
generateRandomIDs(new_entities, old_to_new_ids);
replaceDependencies(new_entities, old_to_new_ids);
std::unordered_set<UUID> unresolved_ids;
return new_entities;
for (const auto & [id, entity_info] : entity_infos)
{
if (entity_info.restore)
ids_to_restore.insert(id);
if (entity_info.new_id)
old_to_new_ids[id] = *entity_info.new_id;
else
unresolved_ids.insert(id);
}
/// Calculate `is_dependent` for each entity info.
if (update_dependents)
{
for (auto & [id, entity_info] : entity_infos)
{
if (!entity_info.restore && entity_info.new_id && entity_info.entity && entity_info.entity->hasDependencies(ids_to_restore))
entity_info.is_dependent = true;
}
}
/// Remap the UUIDs of dependencies in the access entities we're going to restore.
for (auto & [id, entity_info] : entity_infos)
{
if (entity_info.restore || entity_info.is_dependent)
{
auto new_entity = entity_info.entity->clone();
new_entity->replaceDependencies(old_to_new_ids);
new_entity->removeDependencies(unresolved_ids);
entity_info.entity = new_entity;
}
if (entity_info.restore && data_path_with_entities_to_restore.empty())
data_path_with_entities_to_restore = data_paths_in_backup[entity_info.data_path_index];
}
ids_assigned = true;
}
AccessEntitiesToRestore AccessRestorerFromBackup::getEntitiesToRestore(const String & data_path_in_backup) const
{
if (!ids_assigned)
throw Exception(ErrorCodes::LOGICAL_ERROR, "IDs not assigned");
if (data_path_in_backup != data_path_with_entities_to_restore)
return {};
AccessEntitiesToRestore res;
res.new_entities.reserve(entity_infos.size());
res.dependents.reserve(entity_infos.size());
for (const auto & [id, entity_info] : entity_infos)
{
if (entity_info.restore)
res.new_entities.emplace_back(*entity_info.new_id, entity_info.entity);
if (entity_info.is_dependent)
res.dependents.emplace_back(AccessEntitiesToRestore::Dependent{*entity_info.new_id, entity_info.entity});
}
return res;
}
void restoreAccessEntitiesFromBackup(
IAccessStorage & destination_access_storage,
const AccessEntitiesToRestore & entities_to_restore,
const RestoreSettings & restore_settings)
{
if (entities_to_restore.new_entities.empty())
return; /// Nothing to restore.
auto log = getLogger("AccessRestorerFromBackup");
bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace);
bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate);
bool update_dependents = restore_settings.update_access_entities_dependents;
std::unordered_set<UUID> restored_ids;
std::unordered_map<UUID, UUID> new_to_existing_ids;
AccessEntitiesToRestore::Dependents additional_dependents;
additional_dependents.reserve(entities_to_restore.new_entities.size());
for (const auto & [id, entity] : entities_to_restore.new_entities)
{
const String & name = entity->getName();
auto type = entity->getType();
LOG_TRACE(log, "{}: Adding with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), id);
UUID existing_id;
if (destination_access_storage.insert(id, entity, replace_if_exists, throw_if_exists, &existing_id))
{
LOG_TRACE(log, "{}: Added successfully", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name));
restored_ids.emplace(id);
}
else
{
/// Couldn't insert `entity` because there is an existing entity with the same name.
LOG_TRACE(log, "{}: Not added because already exists with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), existing_id);
new_to_existing_ids[id] = existing_id;
if (update_dependents)
additional_dependents.emplace_back(AccessEntitiesToRestore::Dependent{existing_id, entity});
}
}
if (!new_to_existing_ids.empty())
{
std::vector<UUID> ids_to_update;
ids_to_update.reserve(restored_ids.size());
boost::copy(restored_ids, std::inserter(ids_to_update, ids_to_update.end()));
std::unordered_set<UUID> new_ids;
boost::copy(new_to_existing_ids | boost::adaptors::map_keys, std::inserter(new_ids, new_ids.end()));
/// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
/// then we should correct those dependencies.
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
if (!entity->hasDependencies(new_ids))
return entity;
LOG_TRACE(log, "{}: Updating dependencies", entity->formatTypeWithName());
auto res = entity->clone();
res->replaceDependencies(new_to_existing_ids);
return res;
};
/// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here.
destination_access_storage.tryUpdate(ids_to_update, update_func);
}
auto do_update_dependents = [&](const AccessEntitiesToRestore::Dependents & dependents)
{
if (dependents.empty())
return;
std::vector<UUID> ids_to_update;
ids_to_update.reserve(dependents.size());
std::unordered_map<UUID, AccessEntityPtr> id_to_source;
for (const auto & dependent : dependents)
{
const UUID & id = dependent.existing_id;
if (!destination_access_storage.isReadOnly(id))
{
ids_to_update.emplace_back(id);
auto modified_source = dependent.source->clone();
modified_source->replaceDependencies(new_to_existing_ids);
id_to_source[id] = modified_source;
}
}
/// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
/// then we should correct those dependencies.
auto update_func = [&](const AccessEntityPtr & entity, const UUID & existing_id) -> AccessEntityPtr
{
const auto & source = *id_to_source.at(existing_id);
if (!source.hasDependencies(restored_ids))
return entity;
LOG_TRACE(log, "{}: Updating dependent", entity->formatTypeWithName());
auto res = entity->clone();
res->copyDependenciesFrom(source, restored_ids);
return res;
};
/// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here.
destination_access_storage.tryUpdate(ids_to_update, update_func);
};
do_update_dependents(entities_to_restore.dependents);
do_update_dependents(additional_dependents);
}
}

114
src/Access/AccessBackup.h
View File

@ -1,8 +1,8 @@
#pragma once
#include <Common/Logger.h>
#include <Core/UUID.h>
#include <unordered_map>
#include <unordered_set>
namespace DB
@ -12,6 +12,7 @@ enum class AccessEntityType : uint8_t;
struct IAccessEntity;
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
class AccessRightsElements;
class IAccessStorage;
class IBackup;
using BackupPtr = std::shared_ptr<const IBackup>;
class IBackupEntry;
@ -20,15 +21,42 @@ struct RestoreSettings;
enum class RestoreAccessCreationMode : uint8_t;
/// Makes a backup of access entities of a specified type.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
const String & data_path_in_backup,
size_t counter,
const AccessControl & access_control);
/// Makes a backup entry for of a set of access entities.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
const std::vector<UUID> & entities_ids,
const std::unordered_map<UUID, AccessEntityPtr> & all_entities,
bool write_dependents,
const String & data_path_in_backup);
struct AccessEntitiesToRestore
{
/// Access entities loaded from backup with new randomly generated UUIDs.
std::vector<std::pair<UUID /* new_id */, AccessEntityPtr /* new_entity */>> new_entities;
/// Dependents are access entities which exist already and they should be updated after restoring.
/// For example, if there were a role granted to a user: `CREATE USER user1; CREATE ROLE role1; GRANT role1 TO user1`,
/// and we're restoring only role `role1` because user `user1` already exists,
/// then user `user1` should be modified after restoring role `role1` to add this grant `GRANT role1 TO user1`.
struct Dependent
{
/// UUID of an existing access entities.
UUID existing_id;
/// Source access entity from backup to copy dependencies from.
AccessEntityPtr source;
};
using Dependents = std::vector<Dependent>;
Dependents dependents;
};
/// Restores access entities from a backup.
void restoreAccessEntitiesFromBackup(
IAccessStorage & access_storage,
const AccessEntitiesToRestore & entities_to_restore,
const RestoreSettings & restore_settings);
/// Loads access entities from a backup and prepares them for insertion into an access storage.
class AccessRestorerFromBackup
{
public:
@ -36,21 +64,75 @@ public:
~AccessRestorerFromBackup();
/// Adds a data path to loads access entities from.
void addDataPath(const String & data_path);
void addDataPath(const String & data_path_in_backup);
/// Loads access entities from the backup.
void loadFromBackup();
/// Checks that the current user can do restoring.
/// Function loadFromBackup() must be called before that.
AccessRightsElements getRequiredAccess() const;
/// Inserts all access entities loaded from all the paths added by addDataPath().
std::vector<std::pair<UUID, AccessEntityPtr>> getAccessEntities(const AccessControl & access_control) const;
/// Generates random IDs for access entities we're restoring to insert them into an access storage;
/// and finds IDs of existing access entities which are used as dependencies.
void generateRandomIDsAndResolveDependencies(const AccessControl & access_control);
/// Returns access entities loaded from backup and prepared for insertion into an access storage.
/// Both functions loadFromBackup() and generateRandomIDsAndResolveDependencies() must be called before that.
AccessEntitiesToRestore getEntitiesToRestore(const String & data_path_in_backup) const;
private:
BackupPtr backup;
RestoreAccessCreationMode creation_mode;
bool allow_unresolved_dependencies = false;
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
std::unordered_set<String> data_paths;
const BackupPtr backup;
const RestoreAccessCreationMode creation_mode;
const bool skip_unresolved_dependencies;
const bool update_dependents;
const LoggerPtr log;
/// Whether loadFromBackup() finished.
bool loaded = false;
/// Whether generateRandomIDsAndResolveDependencies() finished.
bool ids_assigned = false;
Strings data_paths_in_backup;
String data_path_with_entities_to_restore;
/// Information about an access entity loaded from the backup.
struct EntityInfo
{
UUID id;
String name;
AccessEntityType type;
AccessEntityPtr entity = nullptr; /// Can be nullptr if `restore=false`.
/// Index in `data_paths_in_backup`.
size_t data_path_index = 0;
/// Whether we're going to restore this entity.
/// For example,
/// in case of `RESTORE TABLE system.roles` this flag is true for all the roles loaded from the backup, and
/// in case of `RESTORE ALL` this flag is always true.
bool restore = false;
/// Whether this entity info was added as a dependency of another entity which we're going to restore.
/// For example, if we're going to restore the following user: `CREATE USER user1 DEFAULT ROLE role1, role2 SETTINGS PROFILE profile1, profile2`
/// then `restore=true` for `user1` and `is_dependency=true` for `role1`, `role2`, `profile1`, `profile2`.
/// Flags `restore` and `is_dependency` both can be set at the same time.
bool is_dependency = false;
/// Whether this entity info is a dependent of another entity which we're going to restore.
/// For example, if we're going to restore role `role1` and there is also the following user stored in the backup:
/// `CREATE USER user1 DEFAULT ROLE role1`, then `is_dependent=true` for `user1`.
/// This flags is set by generateRandomIDsAndResolveDependencies().
bool is_dependent = false;
/// New UUID for this entity - either randomly generated or copied from an existing entity.
/// This UUID is assigned by generateRandomIDsAndResolveDependencies().
std::optional<UUID> new_id = std::nullopt;
};
std::unordered_map<UUID, EntityInfo> entity_infos;
};
}

4
src/Access/AccessControl.cpp
View File

@ -629,9 +629,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
}
}
void AccessControl::restoreFromBackup(RestorerFromBackup & restorer)
void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
{
MultipleAccessStorage::restoreFromBackup(restorer);
MultipleAccessStorage::restoreFromBackup(restorer, data_path_in_backup);
changes_notifier->sendNotifications();
}

2
src/Access/AccessControl.h
View File

@ -124,7 +124,7 @@ public:
AuthResult authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const String & forwarded_address) const;
/// Makes a backup of access entities.
void restoreFromBackup(RestorerFromBackup & restorer) override;
void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;
void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);

2
src/Access/DiskAccessStorage.cpp
View File

@ -676,7 +676,7 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
if (!entry.entity)
entry.entity = readAccessEntityFromDisk(id);
auto old_entity = entry.entity;
auto new_entity = update_func(old_entity);
auto new_entity = update_func(old_entity, id);
if (!new_entity->isTypeOf(old_entity->getType()))
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());

62
src/Access/GrantedRoles.cpp
View File

@ -176,6 +176,16 @@ std::vector<UUID> GrantedRoles::findDependencies() const
return res;
}
bool GrantedRoles::hasDependencies(const std::unordered_set<UUID> & ids) const
{
for (const auto & role_id : roles)
{
if (ids.contains(role_id))
return true;
}
return false;
}
void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
std::vector<UUID> new_ids;
@ -221,4 +231,56 @@ void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & ol
}
}
void GrantedRoles::copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids)
{
bool found = false;
for (const auto & role_id : src.roles)
{
if (ids.contains(role_id))
{
roles.emplace(role_id);
found = true;
}
}
if (found)
{
for (const auto & role_id : src.roles_with_admin_option)
{
if (ids.contains(role_id))
roles_with_admin_option.emplace(role_id);
}
}
}
void GrantedRoles::removeDependencies(const std::unordered_set<UUID> & ids)
{
bool found = false;
for (auto it = roles.begin(); it != roles.end();)
{
if (ids.contains(*it))
{
it = roles.erase(it);
found = true;
}
else
{
++it;
}
}
if (found)
{
for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();)
{
if (ids.contains(*it))
it = roles_with_admin_option.erase(it);
else
++it;
}
}
}
}

3
src/Access/GrantedRoles.h
View File

@ -58,7 +58,10 @@ public:
friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); }
std::vector<UUID> findDependencies() const;
bool hasDependencies(const std::unordered_set<UUID> & ids) const;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
void copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set<UUID> & ids);
void removeDependencies(const std::unordered_set<UUID> & ids);
private:
boost::container::flat_set<UUID> roles;

24
src/Access/IAccessEntity.cpp
View File

@ -9,28 +9,4 @@ bool IAccessEntity::equal(const IAccessEntity & other) const
return (name == other.name) && (getType() == other.getType());
}
void IAccessEntity::replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
if (old_to_new_ids.empty())
return;
bool need_replace_dependencies = false;
auto dependencies = entity->findDependencies();
for (const auto & dependency : dependencies)
{
if (old_to_new_ids.contains(dependency))
{
need_replace_dependencies = true;
break;
}
}
if (!need_replace_dependencies)
return;
auto new_entity = entity->clone();
new_entity->replaceDependencies(old_to_new_ids);
entity = new_entity;
}
}

9
src/Access/IAccessEntity.h
View File

@ -48,10 +48,13 @@ struct IAccessEntity
/// Finds all dependencies.
virtual std::vector<UUID> findDependencies() const { return {}; }
virtual bool hasDependencies(const std::unordered_set<UUID> & /* ids */) const { return false; }
/// Replaces dependencies according to a specified map.
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); }
static void replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids);
virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
virtual void copyDependenciesFrom(const IAccessEntity & /* src */, const std::unordered_set<UUID> & /* ids */) {}
virtual void removeDependencies(const std::unordered_set<UUID> & /* ids */) {}
virtual void clearAllExceptDependencies() {}
/// Whether this access entity should be written to a backup.
virtual bool isBackupAllowed() const { return false; }
@ -67,8 +70,6 @@ protected:
{
return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
}
virtual void doReplaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
};
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;

105
src/Access/IAccessStorage.cpp
View File

@ -4,8 +4,10 @@
#include <Access/User.h>
#include <Access/AccessBackup.h>
#include <Backups/BackupEntriesCollector.h>
#include <Backups/RestorerFromBackup.h>
#include <Backups/IBackupCoordination.h>
#include <Backups/IRestoreCoordination.h>
#include <Backups/RestoreSettings.h>
#include <Backups/RestorerFromBackup.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <Common/callOnce.h>
@ -14,6 +16,7 @@
#include <Poco/UUIDGenerator.h>
#include <Poco/Logger.h>
#include <base/FnTraits.h>
#include <base/range.h>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/range/adaptor/map.hpp>
@ -71,6 +74,18 @@ std::vector<UUID> IAccessStorage::find(AccessEntityType type, const Strings & na
}
std::vector<UUID> IAccessStorage::findAllImpl() const
{
std::vector<UUID> res;
for (auto type : collections::range(AccessEntityType::MAX))
{
auto ids = findAllImpl(type);
res.insert(res.end(), ids.begin(), ids.end());
}
return res;
}
UUID IAccessStorage::getID(AccessEntityType type, const String & name) const
{
auto id = findImpl(type, name);
@ -598,67 +613,59 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c
if (!isBackupAllowed())
throwBackupNotAllowed();
auto entities = readAllWithIDs(type);
std::erase_if(entities, [](const std::pair<UUID, AccessEntityPtr> & x) { return !x.second->isBackupAllowed(); });
if (entities.empty())
auto entities_ids = findAll(type);
if (entities_ids.empty())
return;
auto backup_entry = makeBackupEntryForAccess(
entities,
data_path_in_backup,
backup_entries_collector.getAccessCounter(type),
backup_entries_collector.getContext()->getAccessControl());
auto backup_entry_with_path = makeBackupEntryForAccessEntities(
entities_ids,
backup_entries_collector.getAllAccessEntities(),
backup_entries_collector.getBackupSettings().write_access_entities_dependents,
data_path_in_backup);
backup_entries_collector.addBackupEntry(backup_entry);
if (isReplicated())
{
auto backup_coordination = backup_entries_collector.getBackupCoordination();
auto replication_id = getReplicationID();
backup_coordination->addReplicatedAccessFilePath(replication_id, type, backup_entry_with_path.first);
backup_entries_collector.addPostTask(
[backup_entry = backup_entry_with_path.second,
replication_id,
type,
&backup_entries_collector,
backup_coordination]
{
for (const String & path : backup_coordination->getReplicatedAccessFilePaths(replication_id, type))
backup_entries_collector.addBackupEntry(path, backup_entry);
});
}
else
{
backup_entries_collector.addBackupEntry(backup_entry_with_path);
}
}
void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
{
if (!isRestoreAllowed())
throwRestoreNotAllowed();
if (isReplicated() && !acquireReplicatedRestore(restorer))
return;
auto entities = restorer.getAccessEntitiesToRestore();
if (entities.empty())
return;
auto create_access = restorer.getRestoreSettings().create_access;
bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
restorer.addDataRestoreTask([this, entities_to_restore = std::move(entities), replace_if_exists, throw_if_exists] mutable
if (isReplicated())
{
std::unordered_map<UUID, UUID> new_to_existing_ids;
for (auto & [id, entity] : entities_to_restore)
{
UUID existing_entity_id;
if (!insert(id, entity, replace_if_exists, throw_if_exists, &existing_entity_id))
{
/// Couldn't insert `entity` because there is an existing entity with the same name.
new_to_existing_ids[id] = existing_entity_id;
}
}
auto restore_coordination = restorer.getRestoreCoordination();
if (!restore_coordination->acquireReplicatedAccessStorage(getReplicationID()))
return;
}
if (!new_to_existing_ids.empty())
restorer.addDataRestoreTask(
[this, &restorer, data_path_in_backup]
{
/// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
/// then we should correct those dependencies.
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
{
auto res = entity;
IAccessEntity::replaceDependencies(res, new_to_existing_ids);
return res;
};
std::vector<UUID> ids;
ids.reserve(entities_to_restore.size());
boost::copy(entities_to_restore | boost::adaptors::map_keys, std::back_inserter(ids));
tryUpdate(ids, update_func);
}
});
auto entities_to_restore = restorer.getAccessEntitiesToRestore(data_path_in_backup);
const auto & restore_settings = restorer.getRestoreSettings();
restoreAccessEntitiesFromBackup(*this, entities_to_restore, restore_settings);
});
}

23
src/Access/IAccessStorage.h
View File

@ -66,6 +66,7 @@ public:
/// Returns true if this storage is replicated.
virtual bool isReplicated() const { return false; }
virtual String getReplicationID() const { return ""; }
/// Starts periodic reloading and updating of entities in this storage.
virtual void startPeriodicReloading() {}
@ -90,8 +91,9 @@ public:
/// Returns the identifiers of all the entities of a specified type contained in the storage.
std::vector<UUID> findAll(AccessEntityType type) const;
template <typename EntityClassT>
std::vector<UUID> findAll() const { return findAll(EntityClassT::TYPE); }
/// Returns the identifiers of all the entities in the storage.
template <typename EntityClassT = IAccessEntity>
std::vector<UUID> findAll() const;
/// Searches for an entity with specified type and name. Returns std::nullopt if not found.
std::optional<UUID> find(AccessEntityType type, const String & name) const;
@ -148,7 +150,7 @@ public:
std::optional<std::pair<String, AccessEntityType>> tryReadNameWithType(const UUID & id) const;
/// Reads all entities and returns them with their IDs.
template <typename EntityClassT>
template <typename EntityClassT = IAccessEntity>
std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> readAllWithIDs() const;
std::vector<std::pair<UUID, AccessEntityPtr>> readAllWithIDs(AccessEntityType type) const;
@ -180,7 +182,7 @@ public:
/// Removes multiple entities from the storage. Returns the list of successfully dropped.
std::vector<UUID> tryRemove(const std::vector<UUID> & ids);
using UpdateFunc = std::function<AccessEntityPtr(const AccessEntityPtr &)>;
using UpdateFunc = std::function<AccessEntityPtr(const AccessEntityPtr &, const UUID &)>;
/// Updates an entity stored in the storage. Throws an exception if couldn't update.
bool update(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists = true);
@ -214,11 +216,12 @@ public:
/// Makes a backup of this access storage.
virtual void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const;
virtual void restoreFromBackup(RestorerFromBackup & restorer);
virtual void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup);
protected:
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const = 0;
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
virtual std::vector<UUID> findAllImpl() const;
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id);
@ -267,6 +270,16 @@ private:
};
template <typename EntityClassT>
std::vector<UUID> IAccessStorage::findAll() const
{
if constexpr (std::is_same_v<EntityClassT, IAccessEntity>)
return findAllImpl();
else
return findAllImpl(EntityClassT::TYPE);
}
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::read(const UUID & id, bool throw_if_not_exists) const
{

4
src/Access/LDAPAccessStorage.cpp
View File

@ -163,7 +163,7 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id,
// Update the granted roles of the relevant users.
if (!user_ids.empty())
{
auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_) -> AccessEntityPtr
auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr
{
if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
{
@ -301,7 +301,7 @@ void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String
if (it != external_role_hashes.end() && it->second == external_roles_hash)
return;
auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_) -> AccessEntityPtr
auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr
{
if (auto user = typeid_cast<std::shared_ptr<const User>>(entity_))
{

2
src/Access/MemoryAccessStorage.cpp
View File

@ -204,7 +204,7 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat
Entry & entry = it->second;
auto old_entity = entry.entity;
auto new_entity = update_func(old_entity);
auto new_entity = update_func(old_entity, id);
if (!new_entity->isTypeOf(old_entity->getType()))
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());

6
src/Access/MultipleAccessStorage.cpp
View File

@ -416,7 +416,7 @@ bool MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat
{
if (auto old_entity = storage_for_updating->tryRead(id))
{
auto new_entity = update_func(old_entity);
auto new_entity = update_func(old_entity, id);
if (new_entity->getName() != old_entity->getName())
{
for (const auto & storage : *storages)
@ -508,7 +508,7 @@ void MultipleAccessStorage::backup(BackupEntriesCollector & backup_entries_colle
throwBackupNotAllowed();
}
void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup)
{
auto storages = getStoragesInternal();
@ -516,7 +516,7 @@ void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
{
if (storage->isRestoreAllowed())
{
storage->restoreFromBackup(restorer);
storage->restoreFromBackup(restorer, data_path_in_backup);
return;
}
}

2
src/Access/MultipleAccessStorage.h
View File

@ -59,7 +59,7 @@ public:
bool isBackupAllowed() const override;
bool isRestoreAllowed() const override;
void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
void restoreFromBackup(RestorerFromBackup & restorer) override;
void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override;
bool containsStorage(std::string_view storage_type) const;
protected:

26
src/Access/Quota.cpp
View File

@ -24,9 +24,33 @@ std::vector<UUID> Quota::findDependencies() const
return to_roles.findDependencies();
}
void Quota::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
bool Quota::hasDependencies(const std::unordered_set<UUID> & ids) const
{
return to_roles.hasDependencies(ids);
}
void Quota::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
to_roles.replaceDependencies(old_to_new_ids);
}
void Quota::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
{
if (getType() != src.getType())
return;
const auto & src_quota = typeid_cast<const Quota &>(src);
to_roles.copyDependenciesFrom(src_quota.to_roles, ids);
}
void Quota::removeDependencies(const std::unordered_set<UUID> & ids)
{
to_roles.removeDependencies(ids);
}
void Quota::clearAllExceptDependencies()
{
all_limits.clear();
key_type = QuotaKeyType::NONE;
}
}

7
src/Access/Quota.h
View File

@ -47,7 +47,12 @@ struct Quota : public IAccessEntity
AccessEntityType getType() const override { return TYPE; }
std::vector<UUID> findDependencies() const override;
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
void removeDependencies(const std::unordered_set<UUID> & ids) override;
void clearAllExceptDependencies() override;
bool isBackupAllowed() const override { return true; }
};

46
src/Access/ReplicatedAccessStorage.cpp
View File

@ -4,10 +4,6 @@
#include <Access/ReplicatedAccessStorage.h>
#include <Access/AccessChangesNotifier.h>
#include <Access/AccessBackup.h>
#include <Backups/BackupEntriesCollector.h>
#include <Backups/IBackupCoordination.h>
#include <Backups/IRestoreCoordination.h>
#include <Backups/RestorerFromBackup.h>
#include <IO/ReadHelpers.h>
#include <Interpreters/Context.h>
#include <Common/ZooKeeper/KeeperException.h>
@ -359,7 +355,7 @@ bool ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke
}
const AccessEntityPtr old_entity = deserializeAccessEntity(old_entity_definition, entity_path);
const AccessEntityPtr new_entity = update_func(old_entity);
const AccessEntityPtr new_entity = update_func(old_entity, id);
if (!new_entity->isTypeOf(old_entity->getType()))
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());
@ -684,44 +680,4 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if
return memory_storage.read(id, throw_if_not_exists);
}
void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const
{
if (!isBackupAllowed())
throwBackupNotAllowed();
auto entities = readAllWithIDs(type);
std::erase_if(entities, [](const std::pair<UUID, AccessEntityPtr> & x) { return !x.second->isBackupAllowed(); });
if (entities.empty())
return;
auto backup_entry_with_path = makeBackupEntryForAccess(
entities,
data_path_in_backup,
backup_entries_collector.getAccessCounter(type),
backup_entries_collector.getContext()->getAccessControl());
auto backup_coordination = backup_entries_collector.getBackupCoordination();
backup_coordination->addReplicatedAccessFilePath(zookeeper_path, type, backup_entry_with_path.first);
backup_entries_collector.addPostTask(
[backup_entry = backup_entry_with_path.second,
my_zookeeper_path = zookeeper_path,
type,
&backup_entries_collector,
backup_coordination]
{
for (const String & path : backup_coordination->getReplicatedAccessFilePaths(my_zookeeper_path, type))
backup_entries_collector.addBackupEntry(path, backup_entry);
});
}
bool ReplicatedAccessStorage::acquireReplicatedRestore(RestorerFromBackup & restorer) const
{
auto restore_coordination = restorer.getRestoreCoordination();
return restore_coordination->acquireReplicatedAccessStorage(zookeeper_path);
}
}

4
src/Access/ReplicatedAccessStorage.h
View File

@ -26,7 +26,9 @@ public:
void shutdown() override;
const char * getStorageType() const override { return STORAGE_TYPE; }
bool isReplicated() const override { return true; }
String getReplicationID() const override { return zookeeper_path; }
void startPeriodicReloading() override { startWatchingThread(); }
void stopPeriodicReloading() override { stopWatchingThread(); }
@ -35,7 +37,6 @@ public:
bool exists(const UUID & id) const override;
bool isBackupAllowed() const override { return backup_allowed; }
void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
private:
String zookeeper_path;
@ -80,7 +81,6 @@ private:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
bool acquireReplicatedRestore(RestorerFromBackup & restorer) const override;
mutable std::mutex mutex;
MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex);

28
src/Access/Role.cpp
View File

@ -21,10 +21,36 @@ std::vector<UUID> Role::findDependencies() const
return res;
}
void Role::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
bool Role::hasDependencies(const std::unordered_set<UUID> & ids) const
{
return granted_roles.hasDependencies(ids) || settings.hasDependencies(ids);
}
void Role::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
granted_roles.replaceDependencies(old_to_new_ids);
settings.replaceDependencies(old_to_new_ids);
}
void Role::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
{
if (getType() != src.getType())
return;
const auto & src_role = typeid_cast<const Role &>(src);
granted_roles.copyDependenciesFrom(src_role.granted_roles, ids);
settings.copyDependenciesFrom(src_role.settings, ids);
}
void Role::removeDependencies(const std::unordered_set<UUID> & ids)
{
granted_roles.removeDependencies(ids);
settings.removeDependencies(ids);
}
void Role::clearAllExceptDependencies()
{
access = {};
settings.removeSettingsKeepProfiles();
}
}

7
src/Access/Role.h
View File

@ -21,7 +21,12 @@ struct Role : public IAccessEntity
AccessEntityType getType() const override { return TYPE; }
std::vector<UUID> findDependencies() const override;
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
void removeDependencies(const std::unordered_set<UUID> & ids) override;
void clearAllExceptDependencies() override;
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
};

54
src/Access/RolesOrUsersSet.cpp
View File

@ -295,6 +295,23 @@ std::vector<UUID> RolesOrUsersSet::findDependencies() const
return res;
}
bool RolesOrUsersSet::hasDependencies(const std::unordered_set<UUID> & dependencies_ids) const
{
for (const auto & id : ids)
{
if (dependencies_ids.contains(id))
return true;
}
for (const auto & id : except_ids)
{
if (dependencies_ids.contains(id))
return true;
}
return false;
}
void RolesOrUsersSet::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
std::vector<UUID> new_ids;
@ -337,4 +354,41 @@ void RolesOrUsersSet::replaceDependencies(const std::unordered_map<UUID, UUID> &
boost::range::copy(new_ids, std::inserter(except_ids, except_ids.end()));
}
void RolesOrUsersSet::copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set<UUID> & dependencies_ids)
{
if (all != src.all)
return;
for (const auto & id : src.ids)
{
if (dependencies_ids.contains(id))
ids.emplace(id);
}
for (const auto & id : src.except_ids)
{
if (dependencies_ids.contains(id))
except_ids.emplace(id);
}
}
void RolesOrUsersSet::removeDependencies(const std::unordered_set<UUID> & dependencies_ids)
{
for (auto it = ids.begin(); it != ids.end();)
{
if (dependencies_ids.contains(*it))
it = ids.erase(it);
else
++it;
}
for (auto it = except_ids.begin(); it != except_ids.end();)
{
if (dependencies_ids.contains(*it))
except_ids.erase(it);
else
++it;
}
}
}

3
src/Access/RolesOrUsersSet.h
View File

@ -64,7 +64,10 @@ struct RolesOrUsersSet
friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); }
std::vector<UUID> findDependencies() const;
bool hasDependencies(const std::unordered_set<UUID> & dependencies_ids) const;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
void copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set<UUID> & dependencies_ids);
void removeDependencies(const std::unordered_set<UUID> & dependencies_ids);
bool all = false;
boost::container::flat_set<UUID> ids;

26
src/Access/RowPolicy.cpp
View File

@ -63,9 +63,33 @@ std::vector<UUID> RowPolicy::findDependencies() const
return to_roles.findDependencies();
}
void RowPolicy::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
bool RowPolicy::hasDependencies(const std::unordered_set<UUID> & ids) const
{
return to_roles.hasDependencies(ids);
}
void RowPolicy::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
to_roles.replaceDependencies(old_to_new_ids);
}
void RowPolicy::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
{
if (getType() != src.getType())
return;
const auto & src_policy = typeid_cast<const RowPolicy &>(src);
to_roles.copyDependenciesFrom(src_policy.to_roles, ids);
}
void RowPolicy::removeDependencies(const std::unordered_set<UUID> & ids)
{
to_roles.removeDependencies(ids);
}
void RowPolicy::clearAllExceptDependencies()
{
for (auto & filter : filters)
filter = {};
}
}

7
src/Access/RowPolicy.h
View File

@ -50,7 +50,12 @@ struct RowPolicy : public IAccessEntity
AccessEntityType getType() const override { return TYPE; }
std::vector<UUID> findDependencies() const override;
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
void removeDependencies(const std::unordered_set<UUID> & ids) override;
void clearAllExceptDependencies() override;
bool isBackupAllowed() const override { return true; }
/// Which roles or users should use this row policy.

27
src/Access/SettingsProfile.cpp
View File

@ -21,10 +21,35 @@ std::vector<UUID> SettingsProfile::findDependencies() const
return res;
}
void SettingsProfile::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
bool SettingsProfile::hasDependencies(const std::unordered_set<UUID> & ids) const
{
return elements.hasDependencies(ids) || to_roles.hasDependencies(ids);
}
void SettingsProfile::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
elements.replaceDependencies(old_to_new_ids);
to_roles.replaceDependencies(old_to_new_ids);
}
void SettingsProfile::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
{
if (getType() != src.getType())
return;
const auto & src_profile = typeid_cast<const SettingsProfile &>(src);
elements.copyDependenciesFrom(src_profile.elements, ids);
to_roles.copyDependenciesFrom(src_profile.to_roles, ids);
}
void SettingsProfile::removeDependencies(const std::unordered_set<UUID> & ids)
{
elements.removeDependencies(ids);
to_roles.removeDependencies(ids);
}
void SettingsProfile::clearAllExceptDependencies()
{
elements.removeSettingsKeepProfiles();
}
}

7
src/Access/SettingsProfile.h
View File

@ -22,7 +22,12 @@ struct SettingsProfile : public IAccessEntity
AccessEntityType getType() const override { return TYPE; }
std::vector<UUID> findDependencies() const override;
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
void removeDependencies(const std::unordered_set<UUID> & ids) override;
void clearAllExceptDependencies() override;
bool isBackupAllowed() const override { return elements.isBackupAllowed(); }
};

44
src/Access/SettingsProfileElement.cpp
View File

@ -158,6 +158,18 @@ std::vector<UUID> SettingsProfileElements::findDependencies() const
}
bool SettingsProfileElements::hasDependencies(const std::unordered_set<UUID> & ids) const
{
std::vector<UUID> res;
for (const auto & element : *this)
{
if (element.parent_profile && ids.contains(*element.parent_profile))
return true;
}
return false;
}
void SettingsProfileElements::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
for (auto & element : *this)
@ -176,6 +188,38 @@ void SettingsProfileElements::replaceDependencies(const std::unordered_map<UUID,
}
void SettingsProfileElements::copyDependenciesFrom(const SettingsProfileElements & src, const std::unordered_set<UUID> & ids)
{
SettingsProfileElements new_elements;
for (const auto & element : src)
{
if (element.parent_profile && ids.contains(*element.parent_profile))
{
SettingsProfileElement new_element;
new_element.parent_profile = *element.parent_profile;
new_elements.emplace_back(new_element);
}
}
insert(begin(), new_elements.begin(), new_elements.end());
}
void SettingsProfileElements::removeDependencies(const std::unordered_set<UUID> & ids)
{
std::erase_if(
*this, [&](const SettingsProfileElement & element) { return element.parent_profile && ids.contains(*element.parent_profile); });
}
void SettingsProfileElements::removeSettingsKeepProfiles()
{
for (auto & element : *this)
element.setting_name.clear();
std::erase_if(*this, [&](const SettingsProfileElement & element) { return element.setting_name.empty() && !element.parent_profile; });
}
void SettingsProfileElements::merge(const SettingsProfileElements & other)
{
insert(end(), other.begin(), other.end());

5
src/Access/SettingsProfileElement.h
View File

@ -63,7 +63,12 @@ public:
std::shared_ptr<ASTSettingsProfileElements> toASTWithNames(const AccessControl & access_control) const;
std::vector<UUID> findDependencies() const;
bool hasDependencies(const std::unordered_set<UUID> & ids) const;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
void copyDependenciesFrom(const SettingsProfileElements & src, const std::unordered_set<UUID> & ids);
void removeDependencies(const std::unordered_set<UUID> & ids);
void removeSettingsKeepProfiles();
void merge(const SettingsProfileElements & other);

36
src/Access/User.cpp
View File

@ -49,7 +49,12 @@ std::vector<UUID> User::findDependencies() const
return res;
}
void User::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
bool User::hasDependencies(const std::unordered_set<UUID> & ids) const
{
return default_roles.hasDependencies(ids) || granted_roles.hasDependencies(ids) || grantees.hasDependencies(ids) || settings.hasDependencies(ids);
}
void User::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
{
default_roles.replaceDependencies(old_to_new_ids);
granted_roles.replaceDependencies(old_to_new_ids);
@ -57,4 +62,33 @@ void User::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_n
settings.replaceDependencies(old_to_new_ids);
}
void User::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids)
{
if (getType() != src.getType())
return;
const auto & src_user = typeid_cast<const User &>(src);
default_roles.copyDependenciesFrom(src_user.default_roles, ids);
granted_roles.copyDependenciesFrom(src_user.granted_roles, ids);
grantees.copyDependenciesFrom(src_user.grantees, ids);
settings.copyDependenciesFrom(src_user.settings, ids);
}
void User::removeDependencies(const std::unordered_set<UUID> & ids)
{
default_roles.removeDependencies(ids);
granted_roles.removeDependencies(ids);
grantees.removeDependencies(ids);
settings.removeDependencies(ids);
}
void User::clearAllExceptDependencies()
{
authentication_methods.clear();
allowed_client_hosts = AllowedClientHosts::AnyHostTag{};
access = {};
settings.removeSettingsKeepProfiles();
default_database = {};
valid_until = 0;
}
}

7
src/Access/User.h
View File

@ -32,7 +32,12 @@ struct User : public IAccessEntity
void setName(const String & name_) override;
std::vector<UUID> findDependencies() const override;
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
bool hasDependencies(const std::unordered_set<UUID> & ids) const override;
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set<UUID> & ids) override;
void removeDependencies(const std::unordered_set<UUID> & ids) override;
void clearAllExceptDependencies() override;
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
};

2
src/Backups/BackupCoordinationRemote.cpp
View File

@ -570,7 +570,7 @@ void BackupCoordinationRemote::prepareReplicatedAccess() const
if (replicated_access)
return;
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntitry> file_path_for_access_entities;
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntity> file_path_for_access_entities;
auto holder = with_retries.createRetriesControlHolder("prepareReplicatedAccess");
holder.retries_ctl.retryLoop(
[&, &zk = holder.faulty_zookeeper]()

19
src/Backups/BackupCoordinationReplicatedAccess.cpp
View File

@ -1,5 +1,9 @@
#include <Backups/BackupCoordinationReplicatedAccess.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -7,7 +11,7 @@ namespace DB
BackupCoordinationReplicatedAccess::BackupCoordinationReplicatedAccess() = default;
BackupCoordinationReplicatedAccess::~BackupCoordinationReplicatedAccess() = default;
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntitry && file_path_for_access_entity)
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntity && file_path_for_access_entity)
{
const auto & access_zk_path = file_path_for_access_entity.access_zk_path;
const auto & access_entity_type = file_path_for_access_entity.access_entity_type;
@ -28,10 +32,19 @@ Strings BackupCoordinationReplicatedAccess::getFilePaths(const String & access_z
return {};
const auto & file_paths = it->second;
if (file_paths.host_to_store_access != host_id)
if ((file_paths.host_to_store_access != host_id) || file_paths.file_paths.empty())
return {};
Strings res{file_paths.file_paths.begin(), file_paths.file_paths.end()};
/// Use the same filename for all the paths in backup.
/// Those filenames have format "access-<UUID>.txt", where UUID is random.
/// It's not really necessary, however it looks better if those files have the same filename
/// for a backup of ReplicatedAccessStorage on different hosts.
Strings res;
res.reserve(file_paths.file_paths.size());
String filename = fs::path{*file_paths.file_paths.begin()}.filename();
for (const auto & file_path : file_paths.file_paths)
res.emplace_back(fs::path{file_path}.replace_filename(filename));
return res;
}

8
src/Backups/BackupCoordinationReplicatedAccess.h
View File

@ -2,7 +2,7 @@
#include <Core/Types.h>
#include <map>
#include <unordered_set>
#include <set>
namespace DB
@ -28,7 +28,7 @@ public:
BackupCoordinationReplicatedAccess();
~BackupCoordinationReplicatedAccess();
struct FilePathForAccessEntitry
struct FilePathForAccessEntity
{
String access_zk_path;
AccessEntityType access_entity_type;
@ -37,7 +37,7 @@ public:
};
/// Adds a path to access*.txt file keeping access entities of a ReplicatedAccessStorage.
void addFilePath(FilePathForAccessEntitry && file_path_for_access_entity);
void addFilePath(FilePathForAccessEntity && file_path_for_access_entity);
/// Returns all paths added by addFilePath() if `host_id` is a host chosen to store access.
Strings getFilePaths(const String & access_zk_path, AccessEntityType access_entity_type, const String & host_id) const;
@ -47,7 +47,7 @@ private:
struct FilePathsAndHost
{
std::unordered_set<String> file_paths;
std::set<String> file_paths;
String host_to_store_access;
};

3
src/Backups/BackupCoordinationStage.h
View File

@ -32,6 +32,9 @@ namespace BackupCoordinationStage
/// Finding databases and tables in the backup which we're going to restore.
constexpr const char * FINDING_TABLES_IN_BACKUP = "finding tables in backup";
/// Loading system access tables and then checking if the current user has enough access to restore.
constexpr const char * CHECKING_ACCESS_RIGHTS = "checking access rights";
/// Creating databases or finding them and checking their definitions.
constexpr const char * CREATING_DATABASES = "creating databases";

16
src/Backups/BackupEntriesCollector.cpp
View File

@ -1,4 +1,5 @@
#include <Access/Common/AccessEntityType.h>
#include <Access/AccessControl.h>
#include <Backups/BackupCoordinationStage.h>
#include <Backups/BackupEntriesCollector.h>
#include <Backups/BackupEntryFromMemory.h>
@ -910,11 +911,20 @@ void BackupEntriesCollector::runPostTasks()
LOG_TRACE(log, "All post tasks successfully executed");
}
size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type)
std::unordered_map<UUID, AccessEntityPtr> BackupEntriesCollector::getAllAccessEntities()
{
std::lock_guard lock(mutex);
access_counters.resize(static_cast<size_t>(AccessEntityType::MAX));
return access_counters[static_cast<size_t>(type)]++;
if (!all_access_entities)
{
all_access_entities.emplace();
auto entities_with_ids = context->getAccessControl().readAllWithIDs();
for (const auto & [id, entity] : entities_with_ids)
{
if (entity->isBackupAllowed())
all_access_entities->emplace(id, entity);
}
}
return *all_access_entities;
}
}

12
src/Backups/BackupEntriesCollector.h
View File

@ -21,7 +21,8 @@ class IBackupCoordination;
class IDatabase;
using DatabasePtr = std::shared_ptr<IDatabase>;
struct StorageID;
enum class AccessEntityType : uint8_t;
struct IAccessEntity;
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
class QueryStatus;
using QueryStatusPtr = std::shared_ptr<QueryStatus>;
@ -49,6 +50,9 @@ public:
ContextPtr getContext() const { return context; }
const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return global_zookeeper_retries_info; }
/// Returns all access entities which can be put into a backup.
std::unordered_map<UUID, AccessEntityPtr> getAllAccessEntities();
/// Adds a backup entry which will be later returned by run().
/// These function can be called by implementations of IStorage::backupData() in inherited storage classes.
void addBackupEntry(const String & file_name, BackupEntryPtr backup_entry);
@ -61,9 +65,6 @@ public:
/// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts.
void addPostTask(std::function<void()> task);
/// Returns an incremental counter used to backup access control.
size_t getAccessCounter(AccessEntityType type);
private:
void calculateRootPathInBackup();
@ -177,9 +178,10 @@ private:
std::vector<std::pair<String, String>> previous_databases_metadata;
std::vector<std::pair<QualifiedTableName, String>> previous_tables_metadata;
std::optional<std::unordered_map<UUID, AccessEntityPtr>> all_access_entities;
BackupEntries backup_entries;
std::queue<std::function<void()>> post_tasks;
std::vector<size_t> access_counters;
ThreadPool & threadpool;
std::mutex mutex;

1
src/Backups/BackupSettings.cpp
View File

@ -37,6 +37,7 @@ namespace ErrorCodes
M(Bool, check_parts) \
M(Bool, check_projection_parts) \
M(Bool, allow_backup_broken_projections) \
M(Bool, write_access_entities_dependents) \
M(Bool, internal) \
M(String, host_id) \
M(OptionalUUID, backup_uuid)

5
src/Backups/BackupSettings.h
View File

@ -77,6 +77,11 @@ struct BackupSettings
/// Allow to create backup with broken projections.
bool allow_backup_broken_projections = false;
/// Whether dependents of access entities should be written along with the access entities.
/// For example, if a role is granted to a user and we're making a backup of system.roles (but not system.users)
/// this is whether the backup will contain information to grant the role to the corresponding user again.
bool write_access_entities_dependents = true;
/// Internal, should not be specified by user.
/// Whether this backup is a part of a distributed backup created by BACKUP ON CLUSTER.
bool internal = false;

10
src/Backups/RestoreSettings.cpp
View File

@ -160,7 +160,8 @@ namespace
M(UInt64, replica_num_in_backup) \
M(Bool, allow_non_empty_tables) \
M(RestoreAccessCreationMode, create_access) \
M(Bool, allow_unresolved_access_dependencies) \
M(Bool, skip_unresolved_access_dependencies) \
M(Bool, update_access_entities_dependents) \
M(RestoreUDFCreationMode, create_function) \
M(Bool, allow_s3_native_copy) \
M(Bool, use_same_s3_credentials_for_base_backup) \
@ -187,7 +188,12 @@ RestoreSettings RestoreSettings::fromRestoreQuery(const ASTBackupQuery & query)
else
LIST_OF_RESTORE_SETTINGS(GET_SETTINGS_FROM_RESTORE_QUERY_HELPER)
throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Unknown setting {}", setting.name);
/// `allow_unresolved_access_dependencies` is an obsolete name.
if (setting.name == "allow_unresolved_access_dependencies")
res.skip_unresolved_access_dependencies = SettingFieldBool{setting.value}.value;
else
throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Unknown setting {}", setting.name);
}
}

22
src/Backups/RestoreSettings.h
View File

@ -100,9 +100,25 @@ struct RestoreSettings
/// How the RESTORE command will handle if an user (or role or profile) which it's going to restore already exists.
RestoreAccessCreationMode create_access = RestoreAccessCreationMode::kCreateIfNotExists;
/// Skip dependencies of access entities which can't be resolved.
/// For example, if an user has a profile assigned and that profile is not in the backup and doesn't exist locally.
bool allow_unresolved_access_dependencies = false;
/// Ignore dependencies or dependents (with update_access_entities_dependents=true) of access entities in the case if they can't be resolved.
/// For example: if a backup contains a profile assigned to a user: `CREATE PROFILE p1; CREATE USER u1 SETTINGS PROFILE p1`
/// and now we're restoring only user `u1` and profile `p1` doesn't exists, then
/// this flag is whether RESTORE should continue with restoring user `u1` without assigning profile `p1`.
/// Another example: if a backup contains a role granted to a user: `CREATE USER u2; CREATE ROLE r2; GRANT r2 TO u2`
/// and now we're restoring only user `u2` and role `r2` doesn't exist, then
/// this flag is whether RESTORE should continue with restoring user `u2` without that grant.
/// If this flag is false then RESTORE will throw an exception in that case.
bool skip_unresolved_access_dependencies = false;
/// Try to update dependents of restored access entities.
/// For example: if a backup contains a profile assigned to a user: `CREATE PROFILE p1; CREATE USER u1 SETTINGS PROFILE p1`
/// and now we're restoring only profile `p1` and user `u1` already exists, then
/// this flag is whether restored profile `p1` should be assigned to user `u1` again.
/// Another example, if a backup contains a role granted to a user: `CREATE USER u2; CREATE ROLE r2; GRANT r2 TO u2`
/// and now we're restoring only role `r2` and user `u2` already exists, then
/// this flag is whether restored role `r2` should be granted to user `u2` again.
/// If this flag is false then RESTORE won't update existing access entities.
bool update_access_entities_dependents = true;
/// How the RESTORE command will handle if a user-defined function which it's going to restore already exists.
RestoreUDFCreationMode create_function = RestoreUDFCreationMode::kCreateIfNotExists;

64
src/Backups/RestorerFromBackup.cpp
View File

@ -141,6 +141,8 @@ void RestorerFromBackup::run(Mode mode)
waitFutures();
/// Check access rights.
setStage(Stage::CHECKING_ACCESS_RIGHTS);
loadSystemAccessTables();
checkAccessForObjectsFoundInBackup();
if (mode == Mode::CHECK_ACCESS_ONLY)
@ -487,25 +489,6 @@ void RestorerFromBackup::findTableInBackupImpl(const QualifiedTableName & table_
res_table_info.partitions.emplace();
insertAtEnd(*res_table_info.partitions, *partitions);
}
/// Special handling for ACL-related system tables.
if (!restore_settings.structure_only && isSystemAccessTableName(table_name))
{
if (!access_restorer)
access_restorer = std::make_unique<AccessRestorerFromBackup>(backup, restore_settings);
try
{
/// addDataPath() will parse access*.txt files and extract access entities from them.
/// We need to do that early because we need those access entities to check access.
access_restorer->addDataPath(data_path_in_backup);
}
catch (Exception & e)
{
e.addMessage("While parsing data of {} from backup", tableNameWithTypeToString(table_name.database, table_name.table, false));
throw;
}
}
}
void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_backup, const std::set<DatabaseAndTableName> & except_table_names)
@ -629,6 +612,27 @@ size_t RestorerFromBackup::getNumTables() const
return table_infos.size();
}
void RestorerFromBackup::loadSystemAccessTables()
{
if (restore_settings.structure_only)
return;
/// Special handling for ACL-related system tables.
std::lock_guard lock{mutex};
for (const auto & [table_name, table_info] : table_infos)
{
if (isSystemAccessTableName(table_name))
{
if (!access_restorer)
access_restorer = std::make_unique<AccessRestorerFromBackup>(backup, restore_settings);
access_restorer->addDataPath(table_info.data_path_in_backup);
}
}
if (access_restorer)
access_restorer->loadFromBackup();
}
void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const
{
AccessRightsElements required_access;
@ -713,6 +717,15 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const
context->checkAccess(required_access);
}
AccessEntitiesToRestore RestorerFromBackup::getAccessEntitiesToRestore(const String & data_path_in_backup) const
{
std::lock_guard lock{mutex};
if (!access_restorer)
return {};
access_restorer->generateRandomIDsAndResolveDependencies(context->getAccessControl());
return access_restorer->getEntitiesToRestore(data_path_in_backup);
}
void RestorerFromBackup::createDatabases()
{
Strings database_names;
@ -1071,19 +1084,6 @@ void RestorerFromBackup::runDataRestoreTasks()
}
}
std::vector<std::pair<UUID, AccessEntityPtr>> RestorerFromBackup::getAccessEntitiesToRestore()
{
std::lock_guard lock{mutex};
if (!access_restorer || access_restored)
return {};
/// getAccessEntitiesToRestore() will return entities only when called first time (we don't want to restore the same entities again).
access_restored = true;
return access_restorer->getAccessEntities(context->getAccessControl());
}
void RestorerFromBackup::throwTableIsNotEmpty(const StorageID & storage_id)
{
throw Exception(

6
src/Backups/RestorerFromBackup.h
View File

@ -20,8 +20,7 @@ struct StorageID;
class IDatabase;
using DatabasePtr = std::shared_ptr<IDatabase>;
class AccessRestorerFromBackup;
struct IAccessEntity;
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
struct AccessEntitiesToRestore;
class QueryStatus;
using QueryStatusPtr = std::shared_ptr<QueryStatus>;
@ -68,7 +67,7 @@ public:
void addDataRestoreTasks(DataRestoreTasks && new_tasks);
/// Returns the list of access entities to restore.
std::vector<std::pair<UUID, AccessEntityPtr>> getAccessEntitiesToRestore();
AccessEntitiesToRestore getAccessEntitiesToRestore(const String & data_path_in_backup) const;
/// Throws an exception that a specified table is already non-empty.
[[noreturn]] static void throwTableIsNotEmpty(const StorageID & storage_id);
@ -101,6 +100,7 @@ private:
size_t getNumDatabases() const;
size_t getNumTables() const;
void loadSystemAccessTables();
void checkAccessForObjectsFoundInBackup() const;
void createDatabases();

2
src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp
View File

@ -111,7 +111,7 @@ BlockIO InterpreterCreateQuotaQuery::execute()
if (query.alter)
{
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_quota = typeid_cast<std::shared_ptr<Quota>>(entity->clone());
updateQuotaFromQueryImpl(*updated_quota, query, {}, roles_from_query);

2
src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
View File

@ -74,7 +74,7 @@ BlockIO InterpreterCreateRoleQuery::execute()
if (query.alter)
{
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_role = typeid_cast<std::shared_ptr<Role>>(entity->clone());
updateRoleFromQueryImpl(*updated_role, query, {}, settings_from_query);

2
src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp
View File

@ -88,7 +88,7 @@ BlockIO InterpreterCreateRowPolicyQuery::execute()
Strings names = query.names->toStrings();
if (query.alter)
{
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_policy = typeid_cast<std::shared_ptr<RowPolicy>>(entity->clone());
updateRowPolicyFromQueryImpl(*updated_policy, query, {}, roles_from_query);

2
src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp
View File

@ -90,7 +90,7 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute()
if (query.alter)
{
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_profile = typeid_cast<std::shared_ptr<SettingsProfile>>(entity->clone());
updateSettingsProfileFromQueryImpl(*updated_profile, query, {}, settings_from_query, roles_from_query);

4
src/Interpreters/Access/InterpreterCreateUserQuery.cpp
View File

@ -264,7 +264,7 @@ BlockIO InterpreterCreateUserQuery::execute()
if (query.grantees)
grantees_from_query = RolesOrUsersSet{*query.grantees, access_control};
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_user = typeid_cast<std::shared_ptr<User>>(entity->clone());
updateUserFromQueryImpl(
@ -317,7 +317,7 @@ BlockIO InterpreterCreateUserQuery::execute()
if (query.grantees)
{
RolesOrUsersSet grantees_from_query = RolesOrUsersSet{*query.grantees, access_control};
access_control.update(ids, [&](const AccessEntityPtr & entity) -> AccessEntityPtr
access_control.update(ids, [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_user = typeid_cast<std::shared_ptr<User>>(entity->clone());
updated_user->grantees = grantees_from_query;

2
src/Interpreters/Access/InterpreterGrantQuery.cpp
View File

@ -474,7 +474,7 @@ BlockIO InterpreterGrantQuery::execute()
calculateCurrentGrantRightsWithIntersection(new_rights, current_user_access, elements_to_grant);
/// Update roles and users listed in `grantees`.
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto clone = entity->clone();
if (query.current_grants)

2
src/Interpreters/Access/InterpreterSetRoleQuery.cpp
View File

@ -46,7 +46,7 @@ void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query)
std::vector<UUID> to_users = RolesOrUsersSet{*query.to_users, access_control, getContext()->getUserID()}.getMatchingIDs(access_control);
RolesOrUsersSet roles_from_query{*query.roles, access_control};
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr
{
auto updated_user = typeid_cast<std::shared_ptr<User>>(entity->clone());
updateUserSetDefaultRoles(*updated_user, roles_from_query);

4
src/Storages/System/StorageSystemQuotas.cpp
View File

@ -150,10 +150,10 @@ void StorageSystemQuotas::backupData(
}
void StorageSystemQuotas::restoreDataFromBackup(
RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional<ASTs> & /* partitions */)
RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto & access_control = restorer.getContext()->getAccessControl();
access_control.restoreFromBackup(restorer);
access_control.restoreFromBackup(restorer, data_path_in_backup);
}
}

4
src/Storages/System/StorageSystemRoles.cpp
View File

@ -70,10 +70,10 @@ void StorageSystemRoles::backupData(
}
void StorageSystemRoles::restoreDataFromBackup(
RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional<ASTs> & /* partitions */)
RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto & access_control = restorer.getContext()->getAccessControl();
access_control.restoreFromBackup(restorer);
access_control.restoreFromBackup(restorer, data_path_in_backup);
}
}

4
src/Storages/System/StorageSystemRowPolicies.cpp
View File

@ -160,10 +160,10 @@ void StorageSystemRowPolicies::backupData(
}
void StorageSystemRowPolicies::restoreDataFromBackup(
RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional<ASTs> & /* partitions */)
RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto & access_control = restorer.getContext()->getAccessControl();
access_control.restoreFromBackup(restorer);
access_control.restoreFromBackup(restorer, data_path_in_backup);
}
}

4
src/Storages/System/StorageSystemSettingsProfiles.cpp
View File

@ -101,10 +101,10 @@ void StorageSystemSettingsProfiles::backupData(
}
void StorageSystemSettingsProfiles::restoreDataFromBackup(
RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional<ASTs> & /* partitions */)
RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto & access_control = restorer.getContext()->getAccessControl();
access_control.restoreFromBackup(restorer);
access_control.restoreFromBackup(restorer, data_path_in_backup);
}
}

4
src/Storages/System/StorageSystemUsers.cpp
View File

@ -261,10 +261,10 @@ void StorageSystemUsers::backupData(
}
void StorageSystemUsers::restoreDataFromBackup(
RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional<ASTs> & /* partitions */)
RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto & access_control = restorer.getContext()->getAccessControl();
access_control.restoreFromBackup(restorer);
access_control.restoreFromBackup(restorer, data_path_in_backup);
}
}

7
tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.reference Normal file
View File

@ -0,0 +1,7 @@
CREATE USER user_03231 IDENTIFIED WITH no_password DEFAULT ROLE role_a_03231 SETTINGS custom_x = \'x\'
GRANT role_a_03231 TO user_03231
CREATE ROLE role_a_03231
GRANT INSERT ON *.* TO role_a_03231
GRANT role_b_03231 TO role_a_03231
CREATE ROLE role_b_03231
GRANT SELECT ON *.* TO role_b_03231

30
tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.sh Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env bash
# Tags: no-fasttest, no-parallel
# Tag no-fasttest: we restore from a zip-archived backup here.
# Tag no-parallel: we drop and restore fixed users and roles.
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
# In this test we restore from "/tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip"
backup_name="$($CURDIR/helpers/install_predefined_backup.sh old_backup_without_access_entities_dependents.zip)"
${CLICKHOUSE_CLIENT} -m --query "
DROP USER IF EXISTS user_03231;
DROP ROLE IF EXISTS role_a_03231, role_b_03231;
"
${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM Disk('backups', '${backup_name}') FORMAT Null"
${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER user_03231"
${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR user_03231"
${CLICKHOUSE_CLIENT} --query "SHOW CREATE ROLE role_a_03231"
${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR role_a_03231"
${CLICKHOUSE_CLIENT} --query "SHOW CREATE ROLE role_b_03231"
${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR role_b_03231"
${CLICKHOUSE_CLIENT} -m --query "
DROP USER user_03231;
DROP ROLE role_a_03231, role_b_03231;
"

7
tests/queries/0_stateless/03231_restore_user_with_existing_role.reference
View File

@ -1,6 +1,13 @@
Everything dropped
User dropped
Role dropped
Nothing dropped
Nothing dropped, mode=replace
Nothing dropped, mode=create
ACCESS_ENTITY_ALREADY_EXISTS
Everything dropped, restore system.roles, then system.users
user_a 0
role_b 1
Everything dropped, restore system.users, then system.roles
user_a 1
role_b 0

32
tests/queries/0_stateless/03231_restore_user_with_existing_role.sh
View File

@ -55,14 +55,10 @@ ${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}"
${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null"
do_check
# TODO: Cannot restore a dropped role granted to an existing user. The result after RESTORE ALL below is the following:
# CREATE USER user_a DEFAULT ROLE NONE SETTINGS custom_x = 2; GRANT NONE TO user_a; CREATE ROLE role_b SETTINGS custom_x = 1
# because `role_b` is restored but not granted to existing user `user_a`.
#
# echo "Role dropped"
# ${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}"
# ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null"
# do_check
echo "Role dropped"
${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}"
${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null"
do_check
echo "Nothing dropped"
${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null"
@ -75,3 +71,23 @@ do_check
echo "Nothing dropped, mode=create"
${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} SETTINGS create_access='create' FORMAT Null" 2>&1 | grep -om1 "ACCESS_ENTITY_ALREADY_EXISTS"
do_check
echo "Everything dropped, restore system.roles, then system.users"
${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}"
${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}"
# Here "skip_unresolved_access_dependencies=true" because users don't exist yet and restored roles can't be granted to non-existent users.
${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.roles FROM ${backup_name} SETTINGS skip_unresolved_access_dependencies=true FORMAT Null"
${CLICKHOUSE_CLIENT} --query "SELECT 'user_a', count() FROM system.users WHERE name = '${user_a}'"
${CLICKHOUSE_CLIENT} --query "SELECT 'role_b', count() FROM system.roles WHERE name = '${role_b}'"
${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.users FROM ${backup_name} FORMAT Null"
do_check
echo "Everything dropped, restore system.users, then system.roles"
${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}"
${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}"
# Here "skip_unresolved_access_dependencies=true" because roles don't exist yet and can't be granted to restored users.
${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.users FROM ${backup_name} SETTINGS skip_unresolved_access_dependencies=true FORMAT Null"
${CLICKHOUSE_CLIENT} --query "SELECT 'user_a', count() FROM system.users WHERE name = '${user_a}'"
${CLICKHOUSE_CLIENT} --query "SELECT 'role_b', count() FROM system.roles WHERE name = '${role_b}'"
${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.roles FROM ${backup_name} FORMAT Null"
do_check

BIN
tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip Normal file
View File

Binary file not shown.