thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Progressing with WriteBufferFromBlobStorage

Browse Source
This commit is contained in:
Jakub Kuklis 2021-10-01 08:32:28 +00:00 committed by Jakub Kuklis
parent fb4c1d6686
commit c43b90c58c
4 changed files with 56 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/Disks/BlobStorage/DiskBlobStorage.cpp
View File

@ -49,16 +49,16 @@ public:
};
DiskBlobStorage::DiskBlobStorage(
const String & name_,
const String & remote_fs_root_path_,
const String & metadata_path_,
const String & log_name_,
size_t thread_pool_size_) :
IDiskRemote(name_, remote_fs_root_path_, metadata_path_, log_name_, thread_pool_size_) {}
// DiskBlobStorage::DiskBlobStorage(
// const String & name_,
// const String & remote_fs_root_path_,
// const String & metadata_path_,
// const String & log_name_,
// size_t thread_pool_size_) :
// IDiskRemote(name_, remote_fs_root_path_, metadata_path_, log_name_, thread_pool_size_) {}
DiskBlobStorage::DiskBlobStorage() : IDiskRemote("blob_storage", "https://sadttmpstgeus.blob.core.windows.net/data", "/home/jkuklis/blob_storage", "DiskBlobStorage", 1) {}
// DiskBlobStorage::DiskBlobStorage() : IDiskRemote("blob_storage", "https://sadttmpstgeus.blob.core.windows.net/data", "/home/jkuklis/blob_storage", "DiskBlobStorage", 1) {}
DiskBlobStorage::DiskBlobStorage(
@ -66,9 +66,10 @@ DiskBlobStorage::DiskBlobStorage(
const String & metadata_path_,
const String & /* endpoint_url */,
std::shared_ptr<Azure::Identity::ManagedIdentityCredential> /* managed_identity_credential_ */,
Azure::Storage::Blobs::BlobContainerClient /* blob_container_client_ */,
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
size_t thread_pool_size_) :
IDiskRemote(name_, "" /* or maybe endpoint_url ?*/, metadata_path_, "DiskBlobStorage", thread_pool_size_)
IDiskRemote(name_, "" /* or maybe endpoint_url ?*/, metadata_path_, "DiskBlobStorage", thread_pool_size_),
blob_container_client(blob_container_client_)
{
}
@ -95,7 +96,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskBlobStorage::readFile(
std::unique_ptr<WriteBufferFromFileBase> DiskBlobStorage::writeFile(
const String & path,
size_t,
size_t buf_size,
WriteMode mode)
{
auto metadata = readOrCreateMetaForWriting(path, mode);
@ -105,7 +106,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskBlobStorage::writeFile(
LOG_DEBUG(log, "{} to file by path: {}. Blob Storage path: {}",
mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_path + path), remote_fs_root_path + blob_path);
auto buffer = std::make_unique<WriteBufferFromBlobStorage>();
auto buffer = std::make_unique<WriteBufferFromBlobStorage>(blob_container_client, metadata.remote_fs_root_path + blob_path, buf_size);
return std::make_unique<WriteIndirectBufferFromRemoteFS<WriteBufferFromBlobStorage>>(std::move(buffer), std::move(metadata), path);
}

38
src/Disks/BlobStorage/DiskBlobStorage.h
View File

@ -26,14 +26,14 @@ class DiskBlobStorage final : public IDiskRemote
{
public:
DiskBlobStorage(
const String & name_,
const String & remote_fs_root_path_,
const String & metadata_path_,
const String & log_name_,
size_t thread_pool_size);
// DiskBlobStorage(
// const String & name_,
// const String & remote_fs_root_path_,
// const String & metadata_path_,
// const String & log_name_,
// size_t thread_pool_size);
DiskBlobStorage();
// DiskBlobStorage();
DiskBlobStorage(
const String & name_,
@ -45,30 +45,32 @@ public:
);
std::unique_ptr<ReadBufferFromFileBase> readFile(
const String &,
size_t,
size_t,
size_t,
size_t,
MMappedFileCache *) const override;
const String & path,
size_t buf_size,
size_t estimated_size,
size_t direct_io_threshold,
size_t mmap_threshold,
MMappedFileCache * mmap_cache) const override;
std::unique_ptr<WriteBufferFromFileBase> writeFile(
const String &,
size_t,
WriteMode) override;
const String & path,
size_t buf_size,
WriteMode mode) override;
DiskType::Type getType() const override;
bool supportZeroCopyReplication() const override;
bool checkUniqueId(const String &) const override;
bool checkUniqueId(const String & id) const override;
void removeFromRemoteFS(RemoteFSPathKeeperPtr) override;
void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) override;
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
private:
Azure::Storage::Blobs::BlobContainerClient blob_container_client;
};
}

13
src/IO/WriteBufferFromBlobStorage.cpp
View File

@ -6,11 +6,22 @@
#include <IO/WriteBufferFromBlobStorage.h>
namespace DB
{
WriteBufferFromBlobStorage::WriteBufferFromBlobStorage(
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
const String & blob_path_,
size_t buf_size_) :
blob_container_client(blob_container_client_),
blob_path(blob_path_),
buf_size(buf_size_) {}
void WriteBufferFromBlobStorage::nextImpl() {
std::cout << "WriteBufferFromBlobStorage:nextImpl\n\n\n";
std::cout << "buf_size: " << buf_size << "\n";
// std::cout << "WriteBufferFromBlobStorage:nextImpl\n\n\n";
}
}

12
src/IO/WriteBufferFromBlobStorage.h
View File

@ -8,6 +8,8 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <azure/storage/blobs.hpp>
namespace DB
{
@ -15,9 +17,17 @@ namespace DB
class WriteBufferFromBlobStorage : public BufferWithOwnMemory<WriteBuffer>
{
public:
explicit WriteBufferFromBlobStorage() {}
explicit WriteBufferFromBlobStorage(
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
const String & blob_path_,
size_t buf_size_);
void nextImpl() override;
private:
Azure::Storage::Blobs::BlobContainerClient blob_container_client;
const String blob_path;
size_t buf_size;
};
}