From 9c9adf878be21664e88c6be0a5c5941524494634 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 16 Mar 2015 17:56:12 +0300 Subject: [PATCH] dbms: Server: feature development. [#METR-15090] --- dbms/include/DB/IO/IReadFileOperations.h | 20 +++++++++++++++++ dbms/include/DB/IO/IWriteFileOperations.h | 22 +++++++++++++++++++ dbms/include/DB/IO/ReadBufferAIO.h | 11 +++++----- dbms/include/DB/IO/ReadBufferFromFile.h | 2 +- dbms/include/DB/IO/ReadBufferFromFileBase.h | 18 +++++++++++++++ .../DB/IO/ReadBufferFromFileDescriptor.h | 13 ++++++----- dbms/include/DB/IO/WriteBufferAIO.h | 15 +++++++------ dbms/include/DB/IO/WriteBufferFromFileBase.h | 18 +++++++++++++++ .../DB/IO/WriteBufferFromFileDescriptor.h | 20 +++++++++++------ dbms/src/IO/ReadBufferAIO.cpp | 6 ++--- dbms/src/IO/WriteBufferAIO.cpp | 2 +- 11 files changed, 117 insertions(+), 30 deletions(-) create mode 100644 dbms/include/DB/IO/IReadFileOperations.h create mode 100644 dbms/include/DB/IO/IWriteFileOperations.h create mode 100644 dbms/include/DB/IO/ReadBufferFromFileBase.h create mode 100644 dbms/include/DB/IO/WriteBufferFromFileBase.h diff --git a/dbms/include/DB/IO/IReadFileOperations.h b/dbms/include/DB/IO/IReadFileOperations.h new file mode 100644 index 00000000000..d370494ccda --- /dev/null +++ b/dbms/include/DB/IO/IReadFileOperations.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class IReadFileOperations +{ +public: + virtual ~IReadFileOperations() = default; + virtual off_t seek(off_t off, int whence) = 0; + virtual off_t getPositionInFile() = 0; + virtual std::string getFileName() const noexcept = 0; + virtual int getFD() const noexcept = 0; +}; + +} diff --git a/dbms/include/DB/IO/IWriteFileOperations.h b/dbms/include/DB/IO/IWriteFileOperations.h new file mode 100644 index 00000000000..0e0a0466d5f --- /dev/null +++ b/dbms/include/DB/IO/IWriteFileOperations.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class IWriteFileOperations +{ +public: + virtual ~IWriteFileOperations() = default; + virtual off_t seek(off_t off, int whence) = 0; + virtual off_t getPositionInFile() = 0; + virtual void truncate(off_t length) = 0; + virtual void sync() = 0; + virtual std::string getFileName() const noexcept = 0; + virtual int getFD() const noexcept = 0; +}; + +} diff --git a/dbms/include/DB/IO/ReadBufferAIO.h b/dbms/include/DB/IO/ReadBufferAIO.h index 0630e5f1dc4..78752273364 100644 --- a/dbms/include/DB/IO/ReadBufferAIO.h +++ b/dbms/include/DB/IO/ReadBufferAIO.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -15,7 +16,7 @@ namespace DB /** Класс для асинхронного чтения данных. * Все размеры и смещения должны быть кратны DEFAULT_AIO_FILE_BLOCK_SIZE байтам. */ -class ReadBufferAIO : public BufferWithOwnMemory +class ReadBufferAIO : public ReadBufferFromFileBase { public: ReadBufferAIO(const std::string & filename_, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, int flags_ = -1, mode_t mode_ = 0666, @@ -26,10 +27,10 @@ public: ReadBufferAIO & operator=(const ReadBufferAIO &) = delete; void setMaxBytes(size_t max_bytes_read_); - off_t seek(off_t off, int whence = SEEK_SET); - off_t getPositionInFile(); - std::string getFileName() const noexcept { return filename; } - int getFD() const noexcept { return fd; } + off_t seek(off_t off, int whence) override; + off_t getPositionInFile() override; + std::string getFileName() const noexcept override { return filename; } + int getFD() const noexcept override { return fd; } private: off_t getPositionInFileRelaxed() const noexcept; diff --git a/dbms/include/DB/IO/ReadBufferFromFile.h b/dbms/include/DB/IO/ReadBufferFromFile.h index 69ce0fdfc7f..4ff277ee98a 100644 --- a/dbms/include/DB/IO/ReadBufferFromFile.h +++ b/dbms/include/DB/IO/ReadBufferFromFile.h @@ -33,7 +33,7 @@ public: close(fd); } - virtual std::string getFileName() + std::string getFileName() const noexcept override { return file_name; } diff --git a/dbms/include/DB/IO/ReadBufferFromFileBase.h b/dbms/include/DB/IO/ReadBufferFromFileBase.h new file mode 100644 index 00000000000..3e8c3f63ca1 --- /dev/null +++ b/dbms/include/DB/IO/ReadBufferFromFileBase.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class ReadBufferFromFileBase : public IReadFileOperations, public BufferWithOwnMemory +{ +public: + ReadBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment) + : BufferWithOwnMemory(buf_size, existing_memory, alignment) {} + virtual ~ReadBufferFromFileBase() = default; +}; + +} diff --git a/dbms/include/DB/IO/ReadBufferFromFileDescriptor.h b/dbms/include/DB/IO/ReadBufferFromFileDescriptor.h index b412055f509..909811f57cc 100644 --- a/dbms/include/DB/IO/ReadBufferFromFileDescriptor.h +++ b/dbms/include/DB/IO/ReadBufferFromFileDescriptor.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -18,7 +19,7 @@ namespace DB /** Работает с готовым файловым дескриптором. Не открывает и не закрывает файл. */ -class ReadBufferFromFileDescriptor : public BufferWithOwnMemory +class ReadBufferFromFileDescriptor : public ReadBufferFromFileBase { protected: int fd; @@ -53,22 +54,22 @@ protected: } /// Имя или описание файла - virtual std::string getFileName() + virtual std::string getFileName() const noexcept override { return "(fd = " + toString(fd) + ")"; } public: ReadBufferFromFileDescriptor(int fd_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) - : BufferWithOwnMemory(buf_size, existing_memory, alignment), fd(fd_), pos_in_file(0) {} + : ReadBufferFromFileBase(buf_size, existing_memory, alignment), fd(fd_), pos_in_file(0) {} - int getFD() + int getFD() const noexcept override { return fd; } /// Если offset такой маленький, что мы не выйдем за пределы буфера, настоящий seek по файлу не делается. - off_t seek(off_t offset, int whence = SEEK_SET) + off_t seek(off_t offset, int whence = SEEK_CUR) override { off_t new_pos = offset; if (whence == SEEK_CUR) @@ -99,7 +100,7 @@ public: } } - off_t getPositionInFile() + off_t getPositionInFile() override { return pos_in_file - (working_buffer.end() - pos); } diff --git a/dbms/include/DB/IO/WriteBufferAIO.h b/dbms/include/DB/IO/WriteBufferAIO.h index 274de538124..5ce79db30ce 100644 --- a/dbms/include/DB/IO/WriteBufferAIO.h +++ b/dbms/include/DB/IO/WriteBufferAIO.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -14,7 +15,7 @@ namespace DB /** Класс для асинхронной записи данных. * Все размеры и смещения должны быть кратны DEFAULT_AIO_FILE_BLOCK_SIZE байтам. */ -class WriteBufferAIO : public BufferWithOwnMemory +class WriteBufferAIO : public WriteBufferFromFileBase { public: WriteBufferAIO(const std::string & filename_, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, int flags_ = -1, mode_t mode_ = 0666, @@ -24,12 +25,12 @@ public: WriteBufferAIO(const WriteBufferAIO &) = delete; WriteBufferAIO & operator=(const WriteBufferAIO &) = delete; - off_t seek(off_t off, int whence = SEEK_SET); - off_t getPositionInFile(); - void truncate(off_t length = 0); - void sync(); - std::string getFileName() const noexcept { return filename; } - int getFD() const noexcept { return fd; } + off_t seek(off_t off, int whence = SEEK_SET) override; + off_t getPositionInFile() override; + void truncate(off_t length = 0) override; + void sync() override; + std::string getFileName() const noexcept override { return filename; } + int getFD() const noexcept override { return fd; } private: /// Если в буфере ещё остались данные - запишем их. diff --git a/dbms/include/DB/IO/WriteBufferFromFileBase.h b/dbms/include/DB/IO/WriteBufferFromFileBase.h new file mode 100644 index 00000000000..7a6e582e63f --- /dev/null +++ b/dbms/include/DB/IO/WriteBufferFromFileBase.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class WriteBufferFromFileBase : public IWriteFileOperations, public BufferWithOwnMemory +{ +public: + WriteBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment) + : BufferWithOwnMemory(buf_size, existing_memory, alignment) {} + virtual ~WriteBufferFromFileBase() = default; +}; + +} diff --git a/dbms/include/DB/IO/WriteBufferFromFileDescriptor.h b/dbms/include/DB/IO/WriteBufferFromFileDescriptor.h index 6dd76e9d642..def6f45c0a8 100644 --- a/dbms/include/DB/IO/WriteBufferFromFileDescriptor.h +++ b/dbms/include/DB/IO/WriteBufferFromFileDescriptor.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -16,7 +17,7 @@ namespace DB /** Работает с готовым файловым дескриптором. Не открывает и не закрывает файл. */ -class WriteBufferFromFileDescriptor : public BufferWithOwnMemory +class WriteBufferFromFileDescriptor : public WriteBufferFromFileBase { protected: int fd; @@ -40,14 +41,14 @@ protected: } /// Имя или описание файла - virtual std::string getFileName() + virtual std::string getFileName() const noexcept override { return "(fd = " + toString(fd) + ")"; } public: WriteBufferFromFileDescriptor(int fd_ = -1, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) - : BufferWithOwnMemory(buf_size, existing_memory, alignment), fd(fd_) {} + : WriteBufferFromFileBase(buf_size, existing_memory, alignment), fd(fd_) {} /** Можно вызывать для инициализации, если нужный fd не был передан в конструктор. * Менять fd во время работы нельзя. @@ -69,12 +70,12 @@ public: } } - int getFD() + int getFD() const noexcept override { return fd; } - off_t seek(off_t offset, int whence = SEEK_SET) + off_t seek(off_t offset, int whence = SEEK_SET) override { off_t res = lseek(fd, offset, whence); if (-1 == res) @@ -82,14 +83,19 @@ public: return res; } - void truncate(off_t length = 0) + off_t getPositionInFile() override + { + return seek(0, SEEK_CUR); + } + + void truncate(off_t length = 0) override { int res = ftruncate(fd, length); if (-1 == res) throwFromErrno("Cannot truncate file " + getFileName(), ErrorCodes::CANNOT_TRUNCATE_FILE); } - void sync() + void sync() override { /// Если в буфере ещё остались данные - запишем их. next(); diff --git a/dbms/src/IO/ReadBufferAIO.cpp b/dbms/src/IO/ReadBufferAIO.cpp index 28347f8252c..3cdedbf8d94 100644 --- a/dbms/src/IO/ReadBufferAIO.cpp +++ b/dbms/src/IO/ReadBufferAIO.cpp @@ -11,8 +11,8 @@ namespace DB ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_, mode_t mode_, char * existing_memory_) - : BufferWithOwnMemory(buffer_size_, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE), - fill_buffer(BufferWithOwnMemory(buffer_size_, nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)), + : ReadBufferFromFileBase(buffer_size_, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE), + fill_buffer(BufferWithOwnMemory(buffer_size_, nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)), filename(filename_) { ProfileEvents::increment(ProfileEvents::FileOpen); @@ -64,7 +64,7 @@ void ReadBufferAIO::setMaxBytes(size_t max_bytes_read_) max_bytes_read = max_bytes_read_; } -off_t ReadBufferAIO::seek(off_t off, int whence) +off_t ReadBufferAIO::seek(off_t off, int whence = SEEK_CUR) { if ((off % DEFAULT_AIO_FILE_BLOCK_SIZE) != 0) throw Exception("Invalid offset for ReadBufferAIO::seek", ErrorCodes::AIO_UNALIGNED_SIZE_ERROR); diff --git a/dbms/src/IO/WriteBufferAIO.cpp b/dbms/src/IO/WriteBufferAIO.cpp index 792b169c423..a4f17bfa3ff 100644 --- a/dbms/src/IO/WriteBufferAIO.cpp +++ b/dbms/src/IO/WriteBufferAIO.cpp @@ -12,7 +12,7 @@ namespace DB WriteBufferAIO::WriteBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_, mode_t mode_, char * existing_memory_) - : BufferWithOwnMemory(buffer_size_, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE), + : WriteBufferFromFileBase(buffer_size_, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE), flush_buffer(BufferWithOwnMemory(buffer_size_, nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)), filename(filename_) {