diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cac5b70f489..44c72693e49 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,6 +77,7 @@ add_headers_and_sources(clickhouse_common_io IO) add_headers_and_sources(clickhouse_common_io IO/S3) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) +add_headers_and_sources(dbms Disks/IO) if (USE_SQLITE) add_headers_and_sources(dbms Databases/SQLite) endif() diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 9e3b22fac74..c9ee751b202 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Disks/DiskWebServer.cpp b/src/Disks/DiskWebServer.cpp index 148e34cf9c5..009bae355ce 100644 --- a/src/Disks/DiskWebServer.cpp +++ b/src/Disks/DiskWebServer.cpp @@ -3,17 +3,17 @@ #include #include -#include -#include - #include -#include #include #include #include -#include -#include -#include + +#include +#include +#include +#include +#include +#include #include diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index cb3e1e00277..2867fc5de34 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -1,12 +1,14 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include + #include #include diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index 428de44259c..cf1baafce6c 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp similarity index 93% rename from src/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp rename to src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index 08beed079e8..a5661decaa9 100644 --- a/src/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -1,7 +1,8 @@ #include "AsynchronousReadIndirectBufferFromRemoteFS.h" #include -#include +#include +#include #include @@ -46,7 +47,13 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe , min_bytes_for_seek(min_bytes_for_seek_) { ProfileEvents::increment(ProfileEvents::RemoteFSAsyncBuffers); - buffer_events += impl->getFileName() + " : "; + buffer_events += "Events for buffer: " + impl->getFileName() + " : "; +} + + +String AsynchronousReadIndirectBufferFromRemoteFS::getFileName() const +{ + return impl->getFileName(); } @@ -92,9 +99,9 @@ void AsynchronousReadIndirectBufferFromRemoteFS::prefetch() } -void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t offset) +void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t position) { - buffer_events += "-- Set last offset " + toString(offset) + "--"; + buffer_events += "-- Set last offset " + toString(position) + "--"; if (prefetch_future.valid()) { LOG_DEBUG(&Poco::Logger::get("kssenii"), buffer_events); @@ -108,13 +115,14 @@ void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t off // prefetch_future = {}; } - last_offset = offset; - impl->setReadUntilPosition(offset); + last_offset = position; + impl->setReadUntilPosition(position); } bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() { + LOG_DEBUG(&Poco::Logger::get("kssenii"), buffer_events); /// Everything is already read. if (absolute_position == last_offset) return false; @@ -163,12 +171,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() prefetch_future = {}; - /// TODO: it does not really seem to improve anything to call prefetch() here, - /// but it does not make any worse at the same time. - /// Need to test, it might be useful because in fact sometimes (minority of cases though) - /// we can read without prefetching several times in a row. - prefetch(); - + LOG_DEBUG(&Poco::Logger::get("kssenii"), buffer_events); return size; } diff --git a/src/IO/AsynchronousReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h similarity index 92% rename from src/IO/AsynchronousReadIndirectBufferFromRemoteFS.h rename to src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h index f1cfbfde8f6..6842d62f22a 100644 --- a/src/IO/AsynchronousReadIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h @@ -6,14 +6,14 @@ #include #include -#include -#include #include namespace DB { +class ReadBufferFromRemoteFSGather; + /** * Reads data from S3/HDFS/Web using stored paths in metadata. * This class is an asynchronous version of ReadIndirectBufferFromRemoteFS. @@ -43,7 +43,7 @@ public: off_t getPosition() override { return absolute_position - available(); } - String getFileName() const override { return impl->getFileName(); } + String getFileName() const override; void prefetch() override; diff --git a/src/Disks/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp similarity index 97% rename from src/Disks/ReadBufferFromRemoteFSGather.cpp rename to src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index ff9231bf9d3..c12af031089 100644 --- a/src/Disks/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #if USE_AWS_S3 #include @@ -21,12 +21,6 @@ namespace fs = std::filesystem; namespace DB { -namespace ErrorCodes -{ - extern const int CANNOT_SEEK_THROUGH_FILE; -} - - #if USE_AWS_S3 SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const String & path, size_t offset) const { diff --git a/src/Disks/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h similarity index 100% rename from src/Disks/ReadBufferFromRemoteFSGather.h rename to src/Disks/IO/ReadBufferFromRemoteFSGather.h diff --git a/src/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp similarity index 97% rename from src/IO/ReadBufferFromWebServer.cpp rename to src/Disks/IO/ReadBufferFromWebServer.cpp index 245364a9896..69683edec4c 100644 --- a/src/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int CANNOT_SEEK_THROUGH_FILE; extern const int SEEK_POSITION_OUT_OF_BOUND; + extern const int LOGICAL_ERROR; } @@ -49,7 +50,7 @@ std::unique_ptr ReadBufferFromWebServer::initialize() if (last_offset) { if (last_offset < offset) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyound right offset ({} > {})", offset, last_offset - 1); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, last_offset - 1); headers.emplace_back(std::make_pair("Range", fmt::format("bytes={}-{}", offset, last_offset - 1))); LOG_DEBUG(log, "Reading with range: {}-{}", offset, last_offset); @@ -133,7 +134,7 @@ bool ReadBufferFromWebServer::nextImpl() return false; if (last_offset < offset) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyound right offset ({} > {})", offset, last_offset - 1); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, last_offset - 1); } if (impl) @@ -155,7 +156,7 @@ bool ReadBufferFromWebServer::nextImpl() { /** * impl was initialized before, pass position() to it to make - * sure there is no pending data which was not read, becuase + * sure there is no pending data which was not read, because * this branch means we read sequentially. */ impl->position() = position(); diff --git a/src/IO/ReadBufferFromWebServer.h b/src/Disks/IO/ReadBufferFromWebServer.h similarity index 100% rename from src/IO/ReadBufferFromWebServer.h rename to src/Disks/IO/ReadBufferFromWebServer.h diff --git a/src/IO/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp similarity index 97% rename from src/IO/ReadIndirectBufferFromRemoteFS.cpp rename to src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp index d70d280871a..f0133e0fafa 100644 --- a/src/IO/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp @@ -1,6 +1,6 @@ #include "ReadIndirectBufferFromRemoteFS.h" -#include +#include namespace ProfileEvents diff --git a/src/IO/ReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.h similarity index 100% rename from src/IO/ReadIndirectBufferFromRemoteFS.h rename to src/Disks/IO/ReadIndirectBufferFromRemoteFS.h diff --git a/src/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp similarity index 95% rename from src/IO/ThreadPoolRemoteFSReader.cpp rename to src/Disks/IO/ThreadPoolRemoteFSReader.cpp index 258d20d62e9..976162ab625 100644 --- a/src/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -1,4 +1,4 @@ -#include +#include "ThreadPoolRemoteFSReader.h" #include #include @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/src/IO/ThreadPoolRemoteFSReader.h b/src/Disks/IO/ThreadPoolRemoteFSReader.h similarity index 100% rename from src/IO/ThreadPoolRemoteFSReader.h rename to src/Disks/IO/ThreadPoolRemoteFSReader.h diff --git a/src/IO/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp similarity index 100% rename from src/IO/WriteIndirectBufferFromRemoteFS.cpp rename to src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp diff --git a/src/IO/WriteIndirectBufferFromRemoteFS.h b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h similarity index 100% rename from src/IO/WriteIndirectBufferFromRemoteFS.h rename to src/Disks/IO/WriteIndirectBufferFromRemoteFS.h diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index b6c94ca0802..e04802d6b8c 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -17,17 +17,19 @@ #include #include -#include #include #include -#include #include #include #include #include #include -#include -#include + +#include +#include +#include +#include +#include #include // Y_IGNORE #include // Y_IGNORE