diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f861d408a16..1ba512f7be3 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -606,11 +606,6 @@ M(650, SNAPPY_UNCOMPRESS_FAILED) \ M(651, SNAPPY_COMPRESS_FAILED) \ M(652, NO_HIVEMETASTORE) \ - M(653, NOT_INIT)\ - M(654, DISK_OVERFLOW)\ - M(655, FILE_BROKEN)\ - M(656, END_OF_FILE)\ - M(657, CANNOT_RELEASE)\ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 8744a19fdca..5dd28ebc0ff 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -24,10 +24,7 @@ namespace DB namespace fs = std::filesystem; namespace ErrorCodes { - extern const int OK; extern const int LOGICAL_ERROR; - extern const int NOT_INIT; - extern const int DISK_OVERFLOW; } @@ -45,9 +42,8 @@ std::unique_ptr RemoteReadBuffer::create(ContextPtr context, IRemote { auto remote_path = remote_file_metadata->remote_path; auto remote_read_buffer = std::make_unique(buff_size); - ErrorCodes::ErrorCode error; - std::tie(remote_read_buffer->file_cache_controller, read_buffer, error) = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer); + std::tie(remote_read_buffer->file_cache_controller, read_buffer) = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer); if (remote_read_buffer->file_cache_controller == nullptr) { return read_buffer; @@ -114,17 +110,14 @@ ExternalDataSourceCache & ExternalDataSourceCache::instance() return instance; } -void ExternalDataSourceCache::recoverCachedFilesMetadata( - const fs::path & current_path, - size_t current_depth, - size_t max_depth) +void ExternalDataSourceCache::recoverTask() { - if (current_depth >= max_depth) + std::vector invalid_paths; + for (auto const & group_dir : fs::directory_iterator{root_dir}) { - std::vector invalid_paths; - for (auto const & dir : fs::directory_iterator{current_path}) + for (auto const & cache_dir : fs::directory_iterator{group_dir.path()}) { - String path = dir.path(); + String path = cache_dir.path(); auto cache_controller = RemoteCacheController::recover(path); if (!cache_controller) { @@ -136,22 +129,9 @@ void ExternalDataSourceCache::recoverCachedFilesMetadata( invalid_paths.emplace_back(path); } } - for (auto & path : invalid_paths) - { - fs::remove_all(path); - } - return; } - - for (auto const & dir : fs::directory_iterator{current_path}) - { - recoverCachedFilesMetadata(dir.path(), current_depth + 1, max_depth); - } -} - -void ExternalDataSourceCache::recoverTask() -{ - recoverCachedFilesMetadata(root_dir, 1, 2); + for (auto & path : invalid_paths) + fs::remove_all(path); initialized = true; LOG_INFO(log, "Recovered from directory:{}", root_dir); } @@ -191,14 +171,14 @@ String ExternalDataSourceCache::calculateLocalPath(IRemoteFileMetadataPtr metada return fs::path(root_dir) / hashcode_str.substr(0, 3) / hashcode_str; } -std::tuple, ErrorCodes::ErrorCode> +std::pair> ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer) { // If something is wrong on startup, rollback to read from the original ReadBuffer if (!isInitialized()) { LOG_ERROR(log, "ExternalDataSourceCache has not been initialized"); - return {nullptr, std::move(read_buffer), ErrorCodes::NOT_INIT}; + return {nullptr, std::move(read_buffer)}; } auto remote_path = remote_file_metadata->remote_path; @@ -221,7 +201,7 @@ ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr } else { - return {cache, nullptr, ErrorCodes::OK}; + return {cache, nullptr}; } } @@ -235,10 +215,10 @@ ExternalDataSourceCache::createReader(ContextPtr context, IRemoteFileMetadataPtr LOG_ERROR(log, "Insert the new cache failed. new file size:{}, current total size:{}", remote_file_metadata->file_size, lru_caches->weight()); - return {nullptr, std::move(read_buffer), ErrorCodes::DISK_OVERFLOW}; + return {nullptr, std::move(read_buffer)}; } new_cache->startBackgroundDownload(std::move(read_buffer), context->getSchedulePool()); - return {new_cache, nullptr, ErrorCodes::OK}; + return {new_cache, nullptr}; } } diff --git a/src/Storages/Cache/ExternalDataSourceCache.h b/src/Storages/Cache/ExternalDataSourceCache.h index 24c1bcd8607..48faddd7cce 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.h +++ b/src/Storages/Cache/ExternalDataSourceCache.h @@ -60,7 +60,7 @@ public: inline bool isInitialized() const { return initialized; } - std::tuple, ErrorCodes::ErrorCode> + std::pair> createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer); void updateTotalSize(size_t size) { total_size += size; } @@ -84,10 +84,5 @@ private: BackgroundSchedulePool::TaskHolder recover_task_holder; void recoverTask(); - void recoverCachedFilesMetadata( - const std::filesystem::path & current_path, - size_t current_depth, - size_t max_depth); }; - } diff --git a/src/Storages/Cache/RemoteCacheController.cpp b/src/Storages/Cache/RemoteCacheController.cpp index 01394120cb8..8da9a01fe34 100644 --- a/src/Storages/Cache/RemoteCacheController.cpp +++ b/src/Storages/Cache/RemoteCacheController.cpp @@ -31,6 +31,9 @@ std::shared_ptr RemoteCacheController::recover(const std: auto cache_controller = std::make_shared(nullptr, local_path_, 0); if (cache_controller->file_status != DOWNLOADED) { + // do not load this invalid cached file and clear it. the clear action is in + // ExternalDataSourceCache::recoverTask(), because deleting directories during iteration will + // cause unexpected behaviors LOG_INFO(log, "Recover cached file failed. local path:{}", local_path_.string()); return nullptr; } @@ -45,12 +48,6 @@ std::shared_ptr RemoteCacheController::recover(const std: } if (!cache_controller->file_metadata_ptr) { - // do not load this invalid cached file and clear it. the clear action is in - // ExternalDataSourceCache::recoverCachedFilesMetadata(), because deleting directories during iteration will - // cause unexpected behaviors - LOG_ERROR(log, "Cannot create the metadata class : {}. The cached file is invalid and will be remove. path:{}", - cache_controller->metadata_class, - local_path_.string()); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid metadata class:{}", cache_controller->metadata_class); } ReadBufferFromFile file_readbuffer((local_path_ / "metadata.txt").string());