Change filenames of files with access entities inside a backup:

use "access-<UUID>.txt" instead of "access01.txt", "access02.txt", ...
This commit is contained in:
Vitaly Baranov 2024-09-18 13:13:04 +02:00
parent 11ef73f779
commit bd6b7fb6db
8 changed files with 26 additions and 28 deletions

View File

@ -179,17 +179,16 @@ namespace
}
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
const String & data_path_in_backup,
size_t counter,
const AccessControl & access_control)
{
auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control);
AccessEntitiesInBackup ab;
boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end()));
ab.dependencies = std::move(dependencies);
String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ...
String filename = fmt::format("access-{}.txt", UUIDHelpers::generateV4());
String file_path_in_backup = fs::path{data_path_in_backup} / filename;
return {file_path_in_backup, ab.toBackupEntry()};
}

View File

@ -21,11 +21,10 @@ struct RestoreSettings;
enum class RestoreAccessCreationMode : uint8_t;
/// Makes a backup of access entities of a specified type.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
/// Makes a backup entry for of a set of access entities.
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
const String & data_path_in_backup,
size_t counter,
const AccessControl & access_control);
/// Restores access entities from a backup.

View File

@ -607,10 +607,9 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c
if (entities.empty())
return;
auto backup_entry_with_path = makeBackupEntryForAccess(
auto backup_entry_with_path = makeBackupEntryForAccessEntities(
entities,
data_path_in_backup,
backup_entries_collector.getAccessCounter(type),
backup_entries_collector.getContext()->getAccessControl());
if (isReplicated())

View File

@ -570,7 +570,7 @@ void BackupCoordinationRemote::prepareReplicatedAccess() const
if (replicated_access)
return;
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntitry> file_path_for_access_entities;
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntity> file_path_for_access_entities;
auto holder = with_retries.createRetriesControlHolder("prepareReplicatedAccess");
holder.retries_ctl.retryLoop(
[&, &zk = holder.faulty_zookeeper]()

View File

@ -1,5 +1,9 @@
#include <Backups/BackupCoordinationReplicatedAccess.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -7,7 +11,7 @@ namespace DB
BackupCoordinationReplicatedAccess::BackupCoordinationReplicatedAccess() = default;
BackupCoordinationReplicatedAccess::~BackupCoordinationReplicatedAccess() = default;
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntitry && file_path_for_access_entity)
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntity && file_path_for_access_entity)
{
const auto & access_zk_path = file_path_for_access_entity.access_zk_path;
const auto & access_entity_type = file_path_for_access_entity.access_entity_type;
@ -28,10 +32,19 @@ Strings BackupCoordinationReplicatedAccess::getFilePaths(const String & access_z
return {};
const auto & file_paths = it->second;
if (file_paths.host_to_store_access != host_id)
if ((file_paths.host_to_store_access != host_id) || file_paths.file_paths.empty())
return {};
Strings res{file_paths.file_paths.begin(), file_paths.file_paths.end()};
/// Use the same filename for all the paths in backup.
/// Those filenames have format "access-<UUID>.txt", where UUID is random.
/// It's not really necessary, however it looks better if those files have the same filename
/// for a backup of ReplicatedAccessStorage on different hosts.
Strings res;
res.reserve(file_paths.file_paths.size());
String filename = fs::path{*file_paths.file_paths.begin()}.filename();
for (const auto & file_path : file_paths.file_paths)
res.emplace_back(fs::path{file_path}.replace_filename(filename));
return res;
}

View File

@ -2,7 +2,7 @@
#include <Core/Types.h>
#include <map>
#include <unordered_set>
#include <set>
namespace DB
@ -28,7 +28,7 @@ public:
BackupCoordinationReplicatedAccess();
~BackupCoordinationReplicatedAccess();
struct FilePathForAccessEntitry
struct FilePathForAccessEntity
{
String access_zk_path;
AccessEntityType access_entity_type;
@ -37,7 +37,7 @@ public:
};
/// Adds a path to access*.txt file keeping access entities of a ReplicatedAccessStorage.
void addFilePath(FilePathForAccessEntitry && file_path_for_access_entity);
void addFilePath(FilePathForAccessEntity && file_path_for_access_entity);
/// Returns all paths added by addFilePath() if `host_id` is a host chosen to store access.
Strings getFilePaths(const String & access_zk_path, AccessEntityType access_entity_type, const String & host_id) const;
@ -47,7 +47,7 @@ private:
struct FilePathsAndHost
{
std::unordered_set<String> file_paths;
std::set<String> file_paths;
String host_to_store_access;
};

View File

@ -903,11 +903,4 @@ void BackupEntriesCollector::runPostTasks()
LOG_TRACE(log, "All post tasks successfully executed");
}
size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type)
{
std::lock_guard lock(mutex);
access_counters.resize(static_cast<size_t>(AccessEntityType::MAX));
return access_counters[static_cast<size_t>(type)]++;
}
}

View File

@ -21,7 +21,6 @@ class IBackupCoordination;
class IDatabase;
using DatabasePtr = std::shared_ptr<IDatabase>;
struct StorageID;
enum class AccessEntityType : uint8_t;
class QueryStatus;
using QueryStatusPtr = std::shared_ptr<QueryStatus>;
@ -61,9 +60,6 @@ public:
/// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts.
void addPostTask(std::function<void()> task);
/// Returns an incremental counter used to backup access control.
size_t getAccessCounter(AccessEntityType type);
private:
void calculateRootPathInBackup();
@ -179,7 +175,6 @@ private:
BackupEntries backup_entries;
std::queue<std::function<void()>> post_tasks;
std::vector<size_t> access_counters;
ThreadPool & threadpool;
std::mutex mutex;