thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-30 05:30:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Disk S3 possibility to restore parts to 'detached' directory.

Browse Source
This commit is contained in:
Pavel Kovalenko 2021-03-30 13:20:03 +03:00
parent 496c2e45d0
commit b1ba2cf2bc
2 changed files with 47 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/Disks/S3/DiskS3.cpp
View File

@ -1173,6 +1173,7 @@ struct DiskS3::RestoreInformation
UInt64 revision = LATEST_REVISION;
String source_bucket;
String source_path;
bool detached;
};
void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_information)
@ -1201,6 +1202,12 @@ void DiskS3::readRestoreInformation(DiskS3::RestoreInformation & restore_informa
readText(restore_information.source_path, buffer);
assertChar('\n', buffer);
if (!buffer.hasPendingData())
return;
readBoolText(restore_information.detached, buffer);
assertChar('\n', buffer);
if (buffer.hasPendingData())
throw Exception("Extra information at the end of restore file", ErrorCodes::UNKNOWN_FORMAT);
}
@ -1253,7 +1260,7 @@ void DiskS3::restore()
removeSharedRecursive(root + '/', !cleanup_s3);
restoreFiles(information.source_bucket, information.source_path, information.revision);
restoreFileOperations(information.source_bucket, information.source_path, information.revision);
restoreFileOperations(information.source_bucket, information.source_path, information.revision, information.detached);
Poco::File restore_file(metadata_path + RESTORE_FILE_NAME);
restore_file.remove();
@ -1348,14 +1355,16 @@ void DiskS3::processRestoreFiles(const String & source_bucket, const String & so
}
}
void DiskS3::restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision)
void DiskS3::restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision, bool detached)
{
LOG_INFO(&Poco::Logger::get("DiskS3"), "Starting restore file operations for disk {}", name);
/// Enable recording file operations if we restore to different bucket / path.
send_metadata = bucket != source_bucket || s3_root_path != source_path;
listObjects(source_bucket, source_path + "operations/", [this, &source_bucket, &target_revision](auto list_result)
std::set<String> renames;
listObjects(source_bucket, source_path + "operations/", [this, &source_bucket, &target_revision, &detached, &renames](auto list_result)
{
const String rename = "rename";
const String hardlink = "hardlink";
@ -1389,6 +1398,16 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
{
moveFile(from_path, to_path);
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Revision {}. Restored rename {} -> {}", revision, from_path, to_path);
if (detached)
{
/// We don't need path, which is already renamed.
auto it = renames.find(from_path);
if (it != renames.end())
renames.erase(it);
renames.insert(to_path);
}
}
}
else if (operation == hardlink)
@ -1400,6 +1419,9 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
createDirectories(directoryPath(dst_path));
createHardLink(src_path, dst_path);
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Revision {}. Restored hardlink {} -> {}", revision, src_path, dst_path);
if (detached)
renames.insert(directoryPath(dst_path));
}
}
}
@ -1407,6 +1429,18 @@ void DiskS3::restoreFileOperations(const String & source_bucket, const String &
return true;
});
if (detached)
{
send_metadata = false;
for (const auto & path : renames)
{
auto detached_path = pathToDetached(path);
moveFile(path, detached_path);
LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Move directory to detached {} -> {}", path, detached_path);
}
}
send_metadata = true;
LOG_INFO(&Poco::Logger::get("DiskS3"), "File operations restored for disk {}", name);
@ -1451,4 +1485,10 @@ void DiskS3::onFreeze(const String & path)
revision_file_buf.finalize();
}
String DiskS3::pathToDetached(const String & source_path)
{
Poco::Path path (source_path);
return Poco::Path(path).parent().append(Poco::Path("detached")).append(path.directory(path.depth() - 1)).toString();
}
}

5
src/Disks/S3/DiskS3.h
View File

@ -167,13 +167,16 @@ private:
void readRestoreInformation(RestoreInformation & restore_information);
void restoreFiles(const String & source_bucket, const String & source_path, UInt64 target_revision);
void processRestoreFiles(const String & source_bucket, const String & source_path, std::vector<String> keys);
void restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision);
void restoreFileOperations(const String & source_bucket, const String & source_path, UInt64 target_revision, bool detached);
/// Remove 'path' prefix from 'key' to get relative key.
/// It's needed to store keys to metadata files in RELATIVE_PATHS version.
static String shrinkKey(const String & path, const String & key);
std::tuple<UInt64, String> extractRevisionAndOperationFromKey(const String & key);
/// Forms detached path '../../detached/part_name/' from '../../part_name/'
static String pathToDetached(const String & source_path);
const String name;
std::shared_ptr<Aws::S3::S3Client> client;
std::shared_ptr<S3::ProxyConfiguration> proxy_configuration;