thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 07:01:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't use metadata path in write buffers

Browse Source
This commit is contained in:
alesapin 2022-04-08 15:30:06 +02:00
parent 0db8808f2e
commit ad9fdc1d79
5 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp
View File

@ -109,7 +109,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskAzureBlobStorage::writeFile(
readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; }); readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; });
}; };
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(create_metadata_callback), path); return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(create_metadata_callback), blob_path);
} }

4
src/Disks/HDFS/DiskHDFS.cpp
View File

@ -92,7 +92,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path
{ {
/// Path to store new HDFS object. /// Path to store new HDFS object.
auto file_name = getRandomName(); auto file_name = getRandomName();
auto hdfs_path = remote_fs_root_path + file_name; auto hdfs_path = fs::path(remote_fs_root_path) / file_name;
LOG_TRACE(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", LOG_TRACE(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append",
backQuote(metadata_disk->getPath() + path), hdfs_path); backQuote(metadata_disk->getPath() + path), hdfs_path);
@ -106,7 +106,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path
readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; }); readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; });
}; };
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(create_metadata_callback), path); return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(create_metadata_callback), hdfs_path);
} }

4
src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp
View File

@ -12,10 +12,10 @@ namespace DB
WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS(
std::unique_ptr<WriteBuffer> impl_, std::unique_ptr<WriteBuffer> impl_,
CreateMetadataCallback && create_callback_, CreateMetadataCallback && create_callback_,
const String & metadata_file_path_) const String & remote_path_)
: WriteBufferFromFileDecorator(std::move(impl_)) : WriteBufferFromFileDecorator(std::move(impl_))
, create_metadata_callback(std::move(create_callback_)) , create_metadata_callback(std::move(create_callback_))
, metadata_file_path(metadata_file_path_) , remote_path(remote_path_)
{ {
} }

6
src/Disks/IO/WriteIndirectBufferFromRemoteFS.h
View File

@ -18,17 +18,17 @@ public:
WriteIndirectBufferFromRemoteFS( WriteIndirectBufferFromRemoteFS(
std::unique_ptr<WriteBuffer> impl_, std::unique_ptr<WriteBuffer> impl_,
CreateMetadataCallback && create_callback_, CreateMetadataCallback && create_callback_,
const String & metadata_file_path_); const String & remote_path_);
~WriteIndirectBufferFromRemoteFS() override; ~WriteIndirectBufferFromRemoteFS() override;
String getFileName() const override { return metadata_file_path; } String getFileName() const override { return remote_path; }
private: private:
void finalizeImpl() override; void finalizeImpl() override;
CreateMetadataCallback create_metadata_callback; CreateMetadataCallback create_metadata_callback;
String metadata_file_path; String remote_path;
}; };
} }

4
src/Disks/S3/DiskS3.cpp
View File

@ -280,7 +280,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path,
auto s3_buffer = std::make_unique<WriteBufferFromS3>( auto s3_buffer = std::make_unique<WriteBufferFromS3>(
settings->client, settings->client,
bucket, bucket,
remote_fs_root_path + blob_name, fs::path(remote_fs_root_path) / blob_name,
settings->s3_min_upload_part_size, settings->s3_min_upload_part_size,
settings->s3_upload_part_size_multiply_factor, settings->s3_upload_part_size_multiply_factor,
settings->s3_upload_part_size_multiply_parts_count_threshold, settings->s3_upload_part_size_multiply_parts_count_threshold,
@ -293,7 +293,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path,
readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; });
}; };
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(s3_buffer), std::move(create_metadata_callback), path); return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(s3_buffer), std::move(create_metadata_callback), fs::path(remote_fs_root_path) / blob_name);
} }
void DiskS3::createHardLink(const String & src_path, const String & dst_path) void DiskS3::createHardLink(const String & src_path, const String & dst_path)