mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-30 05:30:51 +00:00
Disk S3 possibility to restore parts to 'detached' directory.
This commit is contained in:
parent
496c2e45d0
commit
b1ba2cf2bc
@ -1173,6 +1173,7 @@ struct DiskS3::RestoreInformation
|
||||
UInt64 revision = LATEST_REVISION;
|
||||
String source_bucket;
|
||||
String source_path;
|
||||
bool detached;
|
||||
};
|
||||
|
||||
void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_information)
|
||||
@ -1201,6 +1202,12 @@ void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_informa
|
||||
readText(restore_information.source_path, buffer);
|
||||
assertChar('\n', buffer);
|
||||
|
||||
if (!buffer.hasPendingData())
|
||||
return;
|
||||
|
||||
readBoolText(restore_information.detached, buffer);
|
||||
assertChar('\n', buffer);
|
||||
|
||||
if (buffer.hasPendingData())
|
||||
throw Exception("Extra information at the end of restore file", ErrorCodes::UNKNOWN_FORMAT);
|
||||
}
|
||||
@ -1253,7 +1260,7 @@ void DiskS3::restore()
|
||||
removeSharedRecursive(root + '/', !cleanup_s3);
|
||||
|
||||
restoreFiles(information.source_bucket, information.source_path, information.revision);
|
||||
restoreFileOperations(information.source_bucket, information.source_path, information.revision);
|
||||
restoreFileOperations(information.source_bucket, information.source_path, information.revision, information.detached);
|
||||
|
||||
Poco::File restore_file(metadata_path + RESTORE_FILE_NAME);
|
||||
restore_file.remove();
|
||||
@ -1348,14 +1355,16 @@ void DiskS3::processRestoreFiles(const String & source_bucket, const String & so
|
||||
}
|
||||
}
|
||||
|
||||
void DiskS3::restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision)
|
||||
void DiskS3::restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision, bool detached)
|
||||
{
|
||||
LOG_INFO(&Poco::Logger::get("DiskS3"), "Starting restore file operations for disk {}", name);
|
||||
|
||||
/// Enable recording file operations if we restore to different bucket / path.
|
||||
send_metadata = bucket != source_bucket || s3_root_path != source_path;
|
||||
|
||||
listObjects(source_bucket, source_path + "operations/", [this, &source_bucket, &target_revision](auto list_result)
|
||||
std::set<String> renames;
|
||||
|
||||
listObjects(source_bucket, source_path + "operations/", [this, &source_bucket, &target_revision, &detached, &renames](auto list_result)
|
||||
{
|
||||
const String rename = "rename";
|
||||
const String hardlink = "hardlink";
|
||||
@ -1389,6 +1398,16 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
|
||||
{
|
||||
moveFile(from_path, to_path);
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Revision {}. Restored rename {} -> {}", revision, from_path, to_path);
|
||||
|
||||
if (detached)
|
||||
{
|
||||
/// We don't need path, which is already renamed.
|
||||
auto it = renames.find(from_path);
|
||||
if (it != renames.end())
|
||||
renames.erase(it);
|
||||
|
||||
renames.insert(to_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (operation == hardlink)
|
||||
@ -1400,6 +1419,9 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
|
||||
createDirectories(directoryPath(dst_path));
|
||||
createHardLink(src_path, dst_path);
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Revision {}. Restored hardlink {} -> {}", revision, src_path, dst_path);
|
||||
|
||||
if (detached)
|
||||
renames.insert(directoryPath(dst_path));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1407,6 +1429,18 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
|
||||
return true;
|
||||
});
|
||||
|
||||
if (detached)
|
||||
{
|
||||
send_metadata = false;
|
||||
|
||||
for (const auto & path : renames)
|
||||
{
|
||||
auto detached_path = pathToDetached(path);
|
||||
moveFile(path, detached_path);
|
||||
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Move directory to detached {} -> {}", path, detached_path);
|
||||
}
|
||||
}
|
||||
|
||||
send_metadata = true;
|
||||
|
||||
LOG_INFO(&Poco::Logger::get("DiskS3"), "File operations restored for disk {}", name);
|
||||
@ -1451,4 +1485,10 @@ void DiskS3::onFreeze(const String & path)
|
||||
revision_file_buf.finalize();
|
||||
}
|
||||
|
||||
String DiskS3::pathToDetached(const String & source_path)
|
||||
{
|
||||
Poco::Path path (source_path);
|
||||
return Poco::Path(path).parent().append(Poco::Path("detached")).append(path.directory(path.depth() - 1)).toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -167,13 +167,16 @@ private:
|
||||
void readRestoreInformation(RestoreInformation & restore_information);
|
||||
void restoreFiles(const String & source_bucket, const String & source_path, UInt64 target_revision);
|
||||
void processRestoreFiles(const String & source_bucket, const String & source_path, std::vector<String> keys);
|
||||
void restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision);
|
||||
void restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision, bool detached);
|
||||
|
||||
/// Remove 'path' prefix from 'key' to get relative key.
|
||||
/// It's needed to store keys to metadata files in RELATIVE_PATHS version.
|
||||
static String shrinkKey(const String & path, const String & key);
|
||||
std::tuple<UInt64, String> extractRevisionAndOperationFromKey(const String & key);
|
||||
|
||||
/// Forms detached path '../../detached/part_name/' from '../../part_name/'
|
||||
static String pathToDetached(const String & source_path);
|
||||
|
||||
const String name;
|
||||
std::shared_ptr<Aws::S3::S3Client> client;
|
||||
std::shared_ptr<S3::ProxyConfiguration> proxy_configuration;
|
||||
|
Loading…
Reference in New Issue
Block a user