mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Progressing with WriteBufferFromBlobStorage
This commit is contained in:
parent
fb4c1d6686
commit
c43b90c58c
@ -49,16 +49,16 @@ public:
|
||||
};
|
||||
|
||||
|
||||
DiskBlobStorage::DiskBlobStorage(
|
||||
const String & name_,
|
||||
const String & remote_fs_root_path_,
|
||||
const String & metadata_path_,
|
||||
const String & log_name_,
|
||||
size_t thread_pool_size_) :
|
||||
IDiskRemote(name_, remote_fs_root_path_, metadata_path_, log_name_, thread_pool_size_) {}
|
||||
// DiskBlobStorage::DiskBlobStorage(
|
||||
// const String & name_,
|
||||
// const String & remote_fs_root_path_,
|
||||
// const String & metadata_path_,
|
||||
// const String & log_name_,
|
||||
// size_t thread_pool_size_) :
|
||||
// IDiskRemote(name_, remote_fs_root_path_, metadata_path_, log_name_, thread_pool_size_) {}
|
||||
|
||||
|
||||
DiskBlobStorage::DiskBlobStorage() : IDiskRemote("blob_storage", "https://sadttmpstgeus.blob.core.windows.net/data", "/home/jkuklis/blob_storage", "DiskBlobStorage", 1) {}
|
||||
// DiskBlobStorage::DiskBlobStorage() : IDiskRemote("blob_storage", "https://sadttmpstgeus.blob.core.windows.net/data", "/home/jkuklis/blob_storage", "DiskBlobStorage", 1) {}
|
||||
|
||||
|
||||
DiskBlobStorage::DiskBlobStorage(
|
||||
@ -66,9 +66,10 @@ DiskBlobStorage::DiskBlobStorage(
|
||||
const String & metadata_path_,
|
||||
const String & /* endpoint_url */,
|
||||
std::shared_ptr<Azure::Identity::ManagedIdentityCredential> /* managed_identity_credential_ */,
|
||||
Azure::Storage::Blobs::BlobContainerClient /* blob_container_client_ */,
|
||||
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
|
||||
size_t thread_pool_size_) :
|
||||
IDiskRemote(name_, "" /* or maybe endpoint_url ?*/, metadata_path_, "DiskBlobStorage", thread_pool_size_)
|
||||
IDiskRemote(name_, "" /* or maybe endpoint_url ?*/, metadata_path_, "DiskBlobStorage", thread_pool_size_),
|
||||
blob_container_client(blob_container_client_)
|
||||
{
|
||||
|
||||
}
|
||||
@ -95,7 +96,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskBlobStorage::readFile(
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> DiskBlobStorage::writeFile(
|
||||
const String & path,
|
||||
size_t,
|
||||
size_t buf_size,
|
||||
WriteMode mode)
|
||||
{
|
||||
auto metadata = readOrCreateMetaForWriting(path, mode);
|
||||
@ -105,7 +106,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskBlobStorage::writeFile(
|
||||
LOG_DEBUG(log, "{} to file by path: {}. Blob Storage path: {}",
|
||||
mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_path + path), remote_fs_root_path + blob_path);
|
||||
|
||||
auto buffer = std::make_unique<WriteBufferFromBlobStorage>();
|
||||
auto buffer = std::make_unique<WriteBufferFromBlobStorage>(blob_container_client, metadata.remote_fs_root_path + blob_path, buf_size);
|
||||
|
||||
return std::make_unique<WriteIndirectBufferFromRemoteFS<WriteBufferFromBlobStorage>>(std::move(buffer), std::move(metadata), path);
|
||||
}
|
||||
|
@ -26,14 +26,14 @@ class DiskBlobStorage final : public IDiskRemote
|
||||
{
|
||||
public:
|
||||
|
||||
DiskBlobStorage(
|
||||
const String & name_,
|
||||
const String & remote_fs_root_path_,
|
||||
const String & metadata_path_,
|
||||
const String & log_name_,
|
||||
size_t thread_pool_size);
|
||||
// DiskBlobStorage(
|
||||
// const String & name_,
|
||||
// const String & remote_fs_root_path_,
|
||||
// const String & metadata_path_,
|
||||
// const String & log_name_,
|
||||
// size_t thread_pool_size);
|
||||
|
||||
DiskBlobStorage();
|
||||
// DiskBlobStorage();
|
||||
|
||||
DiskBlobStorage(
|
||||
const String & name_,
|
||||
@ -45,30 +45,32 @@ public:
|
||||
);
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readFile(
|
||||
const String &,
|
||||
size_t,
|
||||
size_t,
|
||||
size_t,
|
||||
size_t,
|
||||
MMappedFileCache *) const override;
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> writeFile(
|
||||
const String &,
|
||||
size_t,
|
||||
WriteMode) override;
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
WriteMode mode) override;
|
||||
|
||||
DiskType::Type getType() const override;
|
||||
|
||||
bool supportZeroCopyReplication() const override;
|
||||
|
||||
bool checkUniqueId(const String &) const override;
|
||||
bool checkUniqueId(const String & id) const override;
|
||||
|
||||
void removeFromRemoteFS(RemoteFSPathKeeperPtr) override;
|
||||
void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) override;
|
||||
|
||||
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
|
||||
|
||||
private:
|
||||
|
||||
Azure::Storage::Blobs::BlobContainerClient blob_container_client;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -6,11 +6,22 @@
|
||||
|
||||
#include <IO/WriteBufferFromBlobStorage.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
WriteBufferFromBlobStorage::WriteBufferFromBlobStorage(
|
||||
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
|
||||
const String & blob_path_,
|
||||
size_t buf_size_) :
|
||||
blob_container_client(blob_container_client_),
|
||||
blob_path(blob_path_),
|
||||
buf_size(buf_size_) {}
|
||||
|
||||
|
||||
void WriteBufferFromBlobStorage::nextImpl() {
|
||||
std::cout << "WriteBufferFromBlobStorage:nextImpl\n\n\n";
|
||||
std::cout << "buf_size: " << buf_size << "\n";
|
||||
// std::cout << "WriteBufferFromBlobStorage:nextImpl\n\n\n";
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -8,6 +8,8 @@
|
||||
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <azure/storage/blobs.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -15,9 +17,17 @@ namespace DB
|
||||
class WriteBufferFromBlobStorage : public BufferWithOwnMemory<WriteBuffer>
|
||||
{
|
||||
public:
|
||||
explicit WriteBufferFromBlobStorage() {}
|
||||
explicit WriteBufferFromBlobStorage(
|
||||
Azure::Storage::Blobs::BlobContainerClient blob_container_client_,
|
||||
const String & blob_path_,
|
||||
size_t buf_size_);
|
||||
|
||||
void nextImpl() override;
|
||||
|
||||
private:
|
||||
Azure::Storage::Blobs::BlobContainerClient blob_container_client;
|
||||
const String blob_path;
|
||||
size_t buf_size;
|
||||
};
|
||||
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user