thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add pread_fake_async method

Browse Source
This commit is contained in:
Alexey Milovidov 2021-08-27 03:35:04 +03:00
parent 0316f1cc3e
commit 864d01e564
3 changed files with 41 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/IO/ReadSettings.h
View File

@ -1,35 +1,50 @@
#pragma once
#include <cstddef>
#include <string>
#include <Core/Defines.h>
namespace DB
{
#define FOR_EACH_READ_METHOD(M) \
/** Simple synchronous reads with 'read'. \
Can use direct IO after specified size. Can use prefetch by asking OS to perform readahead. */ \
M(read) \
\
/** Simple synchronous reads with 'pread'. \
In contrast to 'read', shares single file descriptor from multiple threads. \
Can use direct IO after specified size. Can use prefetch by asking OS to perform readahead. */ \
M(pread) \
\
/** Use mmap after specified size or simple synchronous reads with 'pread'. \
Can use prefetch by asking OS to perform readahead. */ \
M(mmap) \
\
/** Checks if data is in page cache with 'preadv2' on modern Linux kernels. \
If data is in page cache, read from the same thread. \
If not, offload IO to separate threadpool. \
Can do prefetch with double buffering. \
Can use specified priorities and limit the number of concurrent reads. */ \
M(pread_threadpool) \
\
/** It's using asynchronous reader with fake backend that in fact synchronous. \
Only used for testing purposes. */ \
M(pread_fake_async) \
enum class ReadMethod
{
/// Simple synchronous reads with 'read'.
/// Can use direct IO after specified size. Can use prefetch by asking OS to perform readahead.
read,
/// Simple synchronous reads with 'pread'.
/// In contrast to 'read', shares single file descriptor from multiple threads.
/// Can use direct IO after specified size. Can use prefetch by asking OS to perform readahead.
pread,
/// Use mmap after specified size or simple synchronous reads with 'pread'.
/// Can use prefetch by asking OS to perform readahead.
mmap,
/// Checks if data is in page cache with 'preadv2' on modern Linux kernels.
/// If data is in page cache, read from the same thread.
/// If not, offload IO to separate threadpool.
/// Can do prefetch with double buffering.
/// Can use specified priorities and limit the number of concurrent reads.
pread_threadpool
#define DEFINE_READ_METHOD(NAME) NAME,
FOR_EACH_READ_METHOD(DEFINE_READ_METHOD)
#undef DEFINE_READ_METHOD
};
const char * toString(ReadMethod read_method);
ReadMethod parseReadMethod(const std::string & name);
class MMappedFileCache;
struct ReadSettings

6
src/IO/createReadBufferFromFileBase.cpp
View File

@ -64,6 +64,12 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
{
res = std::make_unique<ReadBufferFromFilePReadWithCache>(filename, buffer_size, actual_flags, existing_memory, alignment);
}
else if (settings.local_fs_method == ReadMethod::pread_fake_async)
{
static AsynchronousReaderPtr reader = std::make_shared<SynchronousReader>();
res = std::make_unique<AsynchronousReadBufferFromFileWithCache>(
reader, settings.priority, filename, buffer_size, actual_flags, existing_memory, alignment);
}
else if (settings.local_fs_method == ReadMethod::pread_threadpool)
{
static AsynchronousReaderPtr reader = std::make_shared<ThreadPoolReader>(16, 1000000);

11
src/Interpreters/Context.cpp
View File

@ -2697,16 +2697,7 @@ ReadSettings Context::getReadSettings() const
{
ReadSettings res;
if (settings.local_filesystem_read_method.value == "read")
res.local_fs_method = ReadMethod::read;
else if (settings.local_filesystem_read_method.value == "pread")
res.local_fs_method = ReadMethod::pread;
else if (settings.local_filesystem_read_method.value == "mmap")
res.local_fs_method = ReadMethod::mmap;
else if (settings.local_filesystem_read_method.value == "pread_threadpool")
res.local_fs_method = ReadMethod::pread_threadpool;
else
throw Exception(ErrorCodes::UNKNOWN_READ_METHOD, "Unknown read method '{}'", settings.local_filesystem_read_method.value);
res.local_fs_method = parseReadMethod(settings.local_filesystem_read_method.value);
res.local_fs_prefetch = settings.local_filesystem_read_prefetch;
res.remote_fs_prefetch = settings.remote_filesystem_read_prefetch;