thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #38466 from kssenii/follow-uo-iobjectstorage

Browse Source 
Follow up for #38436
This commit is contained in:
Kseniia Sumarokova 2022-06-28 12:06:42 +02:00 committed by GitHub
commit 4a6af1ba81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 14 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
View File

@ -113,7 +113,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), path);
}
void AzureObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const
void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
{
auto client_ptr = client.get();

2
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
View File

@ -73,7 +73,7 @@ public:
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
const WriteSettings & write_settings = {}) override;
void listPrefix(const std::string & path, PathsWithSize & children) const override;
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
/// Remove file. Throws exception if file doesn't exists or it's a directory.
void removeObject(const std::string & path) override;

2
src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp
View File

@ -51,7 +51,7 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf)
remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size());
}
assertChar('\n', buf);
storage_objects[i].relative_path = remote_fs_object_path;
storage_objects[i].path = remote_fs_object_path;
storage_objects[i].bytes_size = remote_fs_object_size;
}

15
src/Disks/ObjectStorages/DiskObjectStorageMetadata.h
View File

@ -12,17 +12,6 @@ namespace DB
struct DiskObjectStorageMetadata
{
private:
struct RelativePathWithSize
{
String relative_path;
size_t bytes_size;
RelativePathWithSize() = default;
RelativePathWithSize(const String & relative_path_, size_t bytes_size_)
: relative_path(relative_path_), bytes_size(bytes_size_) {}
};
/// Metadata file version.
static constexpr uint32_t VERSION_ABSOLUTE_PATHS = 1;
static constexpr uint32_t VERSION_RELATIVE_PATHS = 2;
@ -31,7 +20,7 @@ private:
const std::string & common_metadata_path;
/// Relative paths of blobs.
std::vector<RelativePathWithSize> storage_objects;
RelativePathsWithSize storage_objects;
/// URI
const std::string & remote_fs_root_path;
@ -71,7 +60,7 @@ public:
return remote_fs_root_path;
}
std::vector<RelativePathWithSize> getBlobsRelativePaths() const
RelativePathsWithSize getBlobsRelativePaths() const
{
return storage_objects;
}

5
src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp
View File

@ -1,5 +1,6 @@
#include <Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.h>
#include <Disks/ObjectStorages/DiskObjectStorage.h>
#include <Disks/ObjectStorages/DiskObjectStorageMetadata.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromFile.h>
@ -379,7 +380,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
return true;
};
PathsWithSize children;
RelativePathsWithSize children;
source_object_storage->listPrefix(restore_information.source_path, children);
restore_files(children);
@ -523,7 +524,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
return true;
};
PathsWithSize children;
RelativePathsWithSize children;
source_object_storage->listPrefix(restore_information.source_path + "operations/", children);
restore_file_operations(children);

2
src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
View File

@ -81,7 +81,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
}
void HDFSObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const
void HDFSObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
{
const size_t begin_of_path = path.find('/', path.find("//") + 2);
int32_t num_entries;

2
src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
View File

@ -75,7 +75,7 @@ public:
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
const WriteSettings & write_settings = {}) override;
void listPrefix(const std::string & path, PathsWithSize & children) const override;
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
/// Remove file. Throws exception if file doesn't exists or it's a directory.
void removeObject(const std::string & path) override;

3
src/Disks/ObjectStorages/IObjectStorage.h
View File

@ -41,6 +41,7 @@ struct PathWithSize
/// List of paths with their sizes
using PathsWithSize = std::vector<PathWithSize>;
using RelativePathsWithSize = PathsWithSize;
struct ObjectMetadata
{
@ -65,7 +66,7 @@ public:
virtual bool exists(const std::string & path) const = 0;
/// List on prefix, return children (relative paths) with their sizes.
virtual void listPrefix(const std::string & path, PathsWithSize & children) const = 0;
virtual void listPrefix(const std::string & path, RelativePathsWithSize & children) const = 0;
/// Get object metadata if supported. It should be possible to receive
/// at least size of object

2
src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
View File

@ -195,7 +195,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(s3_buffer), std::move(finalize_callback), path);
}
void S3ObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const
void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
{
auto settings_ptr = s3_settings.get();
auto client_ptr = client.get();

2
src/Disks/ObjectStorages/S3/S3ObjectStorage.h
View File

@ -80,7 +80,7 @@ public:
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
const WriteSettings & write_settings = {}) override;
void listPrefix(const std::string & path, PathsWithSize & children) const override;
void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
/// Remove file. Throws exception if file doesn't exist or it's a directory.
void removeObject(const std::string & path) override;