mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Change filenames of files with access entities inside a backup:
use "access-<UUID>.txt" instead of "access01.txt", "access02.txt", ...
This commit is contained in:
parent
11ef73f779
commit
bd6b7fb6db
@ -179,17 +179,16 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
|
||||
const String & data_path_in_backup,
|
||||
size_t counter,
|
||||
const AccessControl & access_control)
|
||||
{
|
||||
auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control);
|
||||
AccessEntitiesInBackup ab;
|
||||
boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end()));
|
||||
ab.dependencies = std::move(dependencies);
|
||||
String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ...
|
||||
String filename = fmt::format("access-{}.txt", UUIDHelpers::generateV4());
|
||||
String file_path_in_backup = fs::path{data_path_in_backup} / filename;
|
||||
return {file_path_in_backup, ab.toBackupEntry()};
|
||||
}
|
||||
|
@ -21,11 +21,10 @@ struct RestoreSettings;
|
||||
enum class RestoreAccessCreationMode : uint8_t;
|
||||
|
||||
|
||||
/// Makes a backup of access entities of a specified type.
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
|
||||
/// Makes a backup entry for of a set of access entities.
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccessEntities(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
|
||||
const String & data_path_in_backup,
|
||||
size_t counter,
|
||||
const AccessControl & access_control);
|
||||
|
||||
/// Restores access entities from a backup.
|
||||
|
@ -607,10 +607,9 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c
|
||||
if (entities.empty())
|
||||
return;
|
||||
|
||||
auto backup_entry_with_path = makeBackupEntryForAccess(
|
||||
auto backup_entry_with_path = makeBackupEntryForAccessEntities(
|
||||
entities,
|
||||
data_path_in_backup,
|
||||
backup_entries_collector.getAccessCounter(type),
|
||||
backup_entries_collector.getContext()->getAccessControl());
|
||||
|
||||
if (isReplicated())
|
||||
|
@ -570,7 +570,7 @@ void BackupCoordinationRemote::prepareReplicatedAccess() const
|
||||
if (replicated_access)
|
||||
return;
|
||||
|
||||
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntitry> file_path_for_access_entities;
|
||||
std::vector<BackupCoordinationReplicatedAccess::FilePathForAccessEntity> file_path_for_access_entities;
|
||||
auto holder = with_retries.createRetriesControlHolder("prepareReplicatedAccess");
|
||||
holder.retries_ctl.retryLoop(
|
||||
[&, &zk = holder.faulty_zookeeper]()
|
||||
|
@ -1,5 +1,9 @@
|
||||
#include <Backups/BackupCoordinationReplicatedAccess.h>
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -7,7 +11,7 @@ namespace DB
|
||||
BackupCoordinationReplicatedAccess::BackupCoordinationReplicatedAccess() = default;
|
||||
BackupCoordinationReplicatedAccess::~BackupCoordinationReplicatedAccess() = default;
|
||||
|
||||
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntitry && file_path_for_access_entity)
|
||||
void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntity && file_path_for_access_entity)
|
||||
{
|
||||
const auto & access_zk_path = file_path_for_access_entity.access_zk_path;
|
||||
const auto & access_entity_type = file_path_for_access_entity.access_entity_type;
|
||||
@ -28,10 +32,19 @@ Strings BackupCoordinationReplicatedAccess::getFilePaths(const String & access_z
|
||||
return {};
|
||||
|
||||
const auto & file_paths = it->second;
|
||||
if (file_paths.host_to_store_access != host_id)
|
||||
if ((file_paths.host_to_store_access != host_id) || file_paths.file_paths.empty())
|
||||
return {};
|
||||
|
||||
Strings res{file_paths.file_paths.begin(), file_paths.file_paths.end()};
|
||||
/// Use the same filename for all the paths in backup.
|
||||
/// Those filenames have format "access-<UUID>.txt", where UUID is random.
|
||||
/// It's not really necessary, however it looks better if those files have the same filename
|
||||
/// for a backup of ReplicatedAccessStorage on different hosts.
|
||||
Strings res;
|
||||
res.reserve(file_paths.file_paths.size());
|
||||
String filename = fs::path{*file_paths.file_paths.begin()}.filename();
|
||||
for (const auto & file_path : file_paths.file_paths)
|
||||
res.emplace_back(fs::path{file_path}.replace_filename(filename));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <map>
|
||||
#include <unordered_set>
|
||||
#include <set>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -28,7 +28,7 @@ public:
|
||||
BackupCoordinationReplicatedAccess();
|
||||
~BackupCoordinationReplicatedAccess();
|
||||
|
||||
struct FilePathForAccessEntitry
|
||||
struct FilePathForAccessEntity
|
||||
{
|
||||
String access_zk_path;
|
||||
AccessEntityType access_entity_type;
|
||||
@ -37,7 +37,7 @@ public:
|
||||
};
|
||||
|
||||
/// Adds a path to access*.txt file keeping access entities of a ReplicatedAccessStorage.
|
||||
void addFilePath(FilePathForAccessEntitry && file_path_for_access_entity);
|
||||
void addFilePath(FilePathForAccessEntity && file_path_for_access_entity);
|
||||
|
||||
/// Returns all paths added by addFilePath() if `host_id` is a host chosen to store access.
|
||||
Strings getFilePaths(const String & access_zk_path, AccessEntityType access_entity_type, const String & host_id) const;
|
||||
@ -47,7 +47,7 @@ private:
|
||||
|
||||
struct FilePathsAndHost
|
||||
{
|
||||
std::unordered_set<String> file_paths;
|
||||
std::set<String> file_paths;
|
||||
String host_to_store_access;
|
||||
};
|
||||
|
||||
|
@ -903,11 +903,4 @@ void BackupEntriesCollector::runPostTasks()
|
||||
LOG_TRACE(log, "All post tasks successfully executed");
|
||||
}
|
||||
|
||||
size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
access_counters.resize(static_cast<size_t>(AccessEntityType::MAX));
|
||||
return access_counters[static_cast<size_t>(type)]++;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -21,7 +21,6 @@ class IBackupCoordination;
|
||||
class IDatabase;
|
||||
using DatabasePtr = std::shared_ptr<IDatabase>;
|
||||
struct StorageID;
|
||||
enum class AccessEntityType : uint8_t;
|
||||
class QueryStatus;
|
||||
using QueryStatusPtr = std::shared_ptr<QueryStatus>;
|
||||
|
||||
@ -61,9 +60,6 @@ public:
|
||||
/// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts.
|
||||
void addPostTask(std::function<void()> task);
|
||||
|
||||
/// Returns an incremental counter used to backup access control.
|
||||
size_t getAccessCounter(AccessEntityType type);
|
||||
|
||||
private:
|
||||
void calculateRootPathInBackup();
|
||||
|
||||
@ -179,7 +175,6 @@ private:
|
||||
|
||||
BackupEntries backup_entries;
|
||||
std::queue<std::function<void()>> post_tasks;
|
||||
std::vector<size_t> access_counters;
|
||||
|
||||
ThreadPool & threadpool;
|
||||
std::mutex mutex;
|
||||
|
Loading…
Reference in New Issue
Block a user