thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-11 00:42:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

some modifications

Browse Source
This commit is contained in:
lgbo-ustc 2021-12-28 11:26:39 +08:00 committed by liangjiabiao
parent 0a39273fee
commit 1ac697cad2
4 changed files with 17 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Common/ErrorCodes.cpp
View File

@ -606,11 +606,6 @@
M(650, SNAPPY_UNCOMPRESS_FAILED) \
M(651, SNAPPY_COMPRESS_FAILED) \
M(652, NO_HIVEMETASTORE) \
M(653, NOT_INIT)\
M(654, DISK_OVERFLOW)\
M(655, FILE_BROKEN)\
M(656, END_OF_FILE)\
M(657, CANNOT_RELEASE)\
\
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \

44
src/Storages/Cache/ExternalDataSourceCache.cpp
View File

@ -24,10 +24,7 @@ namespace DB
namespace fs = std::filesystem;
namespace ErrorCodes
{
extern const int OK;
extern const int LOGICAL_ERROR;
extern const int NOT_INIT;
extern const int DISK_OVERFLOW;
}
@ -45,9 +42,8 @@ std::unique_ptr<ReadBuffer> RemoteReadBuffer::create(ContextPtr context, IRemote
{
auto remote_path = remote_file_metadata->remote_path;
auto remote_read_buffer = std::make_unique<RemoteReadBuffer>(buff_size);
ErrorCodes::ErrorCode error;
std::tie(remote_read_buffer->file_cache_controller, read_buffer, error) = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer);
std::tie(remote_read_buffer->file_cache_controller, read_buffer) = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer);
if (remote_read_buffer->file_cache_controller == nullptr)
{
return read_buffer;
@ -114,17 +110,14 @@ ExternalDataSourceCache & ExternalDataSourceCache::instance()
return instance;
}
void ExternalDataSourceCache::recoverCachedFilesMetadata(
const fs::path & current_path,
size_t current_depth,
size_t max_depth)
{
if (current_depth >= max_depth)
void ExternalDataSourceCache::recoverTask()
{
std::vector<fs::path> invalid_paths;
for (auto const & dir : fs::directory_iterator{current_path})
for (auto const & group_dir : fs::directory_iterator{root_dir})
{
String path = dir.path();
for (auto const & cache_dir : fs::directory_iterator{group_dir.path()})
{
String path = cache_dir.path();
auto cache_controller = RemoteCacheController::recover(path);
if (!cache_controller)
{
@ -136,22 +129,9 @@ void ExternalDataSourceCache::recoverCachedFilesMetadata(
invalid_paths.emplace_back(path);
}
}
}
for (auto & path : invalid_paths)
{
fs::remove_all(path);
}
return;
}
for (auto const & dir : fs::directory_iterator{current_path})
{
recoverCachedFilesMetadata(dir.path(), current_depth + 1, max_depth);
}
}
void ExternalDataSourceCache::recoverTask()
{
recoverCachedFilesMetadata(root_dir, 1, 2);
initialized = true;
LOG_INFO(log, "Recovered from directory:{}", root_dir);
}
@ -191,14 +171,14 @@ String ExternalDataSourceCache::calculateLocalPath(IRemoteFileMetadataPtr metada
return fs::path(root_dir) / hashcode_str.substr(0, 3) / hashcode_str;
}
std::tuple<RemoteCacheControllerPtr, std::unique_ptr<ReadBuffer>, ErrorCodes::ErrorCode>
std::pair<RemoteCacheControllerPtr, std::unique_ptr<ReadBuffer>>
ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr<ReadBuffer> & read_buffer)
{
// If something is wrong on startup, rollback to read from the original ReadBuffer
if (!isInitialized())
{
LOG_ERROR(log, "ExternalDataSourceCache has not been initialized");
return {nullptr, std::move(read_buffer), ErrorCodes::NOT_INIT};
return {nullptr, std::move(read_buffer)};
}
auto remote_path = remote_file_metadata->remote_path;
@ -221,7 +201,7 @@ ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr
}
else
{
return {cache, nullptr, ErrorCodes::OK};
return {cache, nullptr};
}
}
@ -235,10 +215,10 @@ ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr
LOG_ERROR(log, "Insert the new cache failed. new file size:{}, current total size:{}",
remote_file_metadata->file_size,
lru_caches->weight());
return {nullptr, std::move(read_buffer), ErrorCodes::DISK_OVERFLOW};
return {nullptr, std::move(read_buffer)};
}
new_cache->startBackgroundDownload(std::move(read_buffer), context->getSchedulePool());
return {new_cache, nullptr, ErrorCodes::OK};
return {new_cache, nullptr};
}
}

7
src/Storages/Cache/ExternalDataSourceCache.h
View File

@ -60,7 +60,7 @@ public:
inline bool isInitialized() const { return initialized; }
std::tuple<RemoteCacheControllerPtr, std::unique_ptr<ReadBuffer>, ErrorCodes::ErrorCode>
std::pair<RemoteCacheControllerPtr, std::unique_ptr<ReadBuffer>>
createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr<ReadBuffer> & read_buffer);
void updateTotalSize(size_t size) { total_size += size; }
@ -84,10 +84,5 @@ private:
BackgroundSchedulePool::TaskHolder recover_task_holder;
void recoverTask();
void recoverCachedFilesMetadata(
const std::filesystem::path & current_path,
size_t current_depth,
size_t max_depth);
};
}

9
src/Storages/Cache/RemoteCacheController.cpp
View File

@ -31,6 +31,9 @@ std::shared_ptr<RemoteCacheController> RemoteCacheController::recover(const std:
auto cache_controller = std::make_shared<RemoteCacheController>(nullptr, local_path_, 0);
if (cache_controller->file_status != DOWNLOADED)
{
// do not load this invalid cached file and clear it. the clear action is in
// ExternalDataSourceCache::recoverTask(), because deleting directories during iteration will
// cause unexpected behaviors
LOG_INFO(log, "Recover cached file failed. local path:{}", local_path_.string());
return nullptr;
}
@ -45,12 +48,6 @@ std::shared_ptr<RemoteCacheController> RemoteCacheController::recover(const std:
}
if (!cache_controller->file_metadata_ptr)
{
// do not load this invalid cached file and clear it. the clear action is in
// ExternalDataSourceCache::recoverCachedFilesMetadata(), because deleting directories during iteration will
// cause unexpected behaviors
LOG_ERROR(log, "Cannot create the metadata class : {}. The cached file is invalid and will be remove. path:{}",
cache_controller->metadata_class,
local_path_.string());
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid metadata class:{}", cache_controller->metadata_class);
}
ReadBufferFromFile file_readbuffer((local_path_ / "metadata.txt").string());