mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Merge pull request #16527 from kitaisreal/mmap-read-buffer-from-file-descriptor-use-getpagesize
MMapReadBufferFromFileDescriptor use getpagesize
This commit is contained in:
commit
f82274a722
@ -6,6 +6,7 @@ set (SRCS
|
||||
demangle.cpp
|
||||
getFQDNOrHostName.cpp
|
||||
getMemoryAmount.cpp
|
||||
getPageSize.cpp
|
||||
getThreadId.cpp
|
||||
JSON.cpp
|
||||
LineReader.cpp
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include <stdexcept>
|
||||
#include "common/getMemoryAmount.h"
|
||||
#include "common/getPageSize.h"
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
@ -18,7 +19,7 @@ uint64_t getMemoryAmountOrZero()
|
||||
if (num_pages <= 0)
|
||||
return 0;
|
||||
|
||||
int64_t page_size = sysconf(_SC_PAGESIZE);
|
||||
int64_t page_size = getPageSize();
|
||||
if (page_size <= 0)
|
||||
return 0;
|
||||
|
||||
|
8
base/common/getPageSize.cpp
Normal file
8
base/common/getPageSize.cpp
Normal file
@ -0,0 +1,8 @@
|
||||
#include "common/getPageSize.h"
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
Int64 getPageSize()
|
||||
{
|
||||
return sysconf(_SC_PAGESIZE);
|
||||
}
|
6
base/common/getPageSize.h
Normal file
6
base/common/getPageSize.h
Normal file
@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "common/types.h"
|
||||
|
||||
/// Get memory page size
|
||||
Int64 getPageSize();
|
@ -47,6 +47,7 @@ SRCS(
|
||||
errnoToString.cpp
|
||||
getFQDNOrHostName.cpp
|
||||
getMemoryAmount.cpp
|
||||
getPageSize.cpp
|
||||
getResource.cpp
|
||||
getThreadId.cpp
|
||||
mremap.cpp
|
||||
|
@ -26,6 +26,7 @@
|
||||
#define DISABLE_MREMAP 1
|
||||
#endif
|
||||
#include <common/mremap.h>
|
||||
#include <common/getPageSize.h>
|
||||
|
||||
#include <Common/MemoryTracker.h>
|
||||
#include <Common/Exception.h>
|
||||
@ -59,7 +60,6 @@
|
||||
*/
|
||||
extern const size_t MMAP_THRESHOLD;
|
||||
|
||||
static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
|
||||
static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
|
||||
|
||||
namespace DB
|
||||
@ -194,10 +194,11 @@ private:
|
||||
void * allocNoTrack(size_t size, size_t alignment)
|
||||
{
|
||||
void * buf;
|
||||
size_t mmap_min_alignment = ::getPageSize();
|
||||
|
||||
if (size >= MMAP_THRESHOLD)
|
||||
{
|
||||
if (alignment > MMAP_MIN_ALIGNMENT)
|
||||
if (alignment > mmap_min_alignment)
|
||||
throw DB::Exception(fmt::format("Too large alignment {}: more than page size when allocating {}.",
|
||||
ReadableSize(alignment), ReadableSize(size)), DB::ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
|
@ -83,10 +83,11 @@ private:
|
||||
/// Last contiguous chunk of memory.
|
||||
Chunk * head;
|
||||
size_t size_in_bytes;
|
||||
size_t page_size;
|
||||
|
||||
static size_t roundUpToPageSize(size_t s)
|
||||
static size_t roundUpToPageSize(size_t s, size_t page_size)
|
||||
{
|
||||
return (s + 4096 - 1) / 4096 * 4096;
|
||||
return (s + page_size - 1) / page_size * page_size;
|
||||
}
|
||||
|
||||
/// If chunks size is less than 'linear_growth_threshold', then use exponential growth, otherwise - linear growth
|
||||
@ -113,7 +114,7 @@ private:
|
||||
}
|
||||
|
||||
assert(size_after_grow >= min_next_size);
|
||||
return roundUpToPageSize(size_after_grow);
|
||||
return roundUpToPageSize(size_after_grow, page_size);
|
||||
}
|
||||
|
||||
/// Add next contiguous chunk of memory with size not less than specified.
|
||||
@ -129,7 +130,8 @@ private:
|
||||
public:
|
||||
Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
|
||||
: growth_factor(growth_factor_), linear_growth_threshold(linear_growth_threshold_),
|
||||
head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size())
|
||||
head(new Chunk(initial_size_, nullptr)), size_in_bytes(head->size()),
|
||||
page_size(static_cast<size_t>(::getPageSize()))
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -13,6 +13,8 @@
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <ext/scope_guard.h>
|
||||
|
||||
#include <common/getPageSize.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/formatReadable.h>
|
||||
@ -326,8 +328,6 @@ private:
|
||||
return (x + (rounding - 1)) / rounding * rounding;
|
||||
}
|
||||
|
||||
static constexpr size_t page_size = 4096;
|
||||
|
||||
/// Sizes and addresses of allocated memory will be aligned to specified boundary.
|
||||
static constexpr size_t alignment = 16;
|
||||
|
||||
@ -505,6 +505,7 @@ private:
|
||||
|
||||
/// If nothing was found and total size of allocated chunks plus required size is lower than maximum,
|
||||
/// allocate a new chunk.
|
||||
size_t page_size = static_cast<size_t>(::getPageSize());
|
||||
size_t required_chunk_size = std::max(min_chunk_size, roundUp(size, page_size));
|
||||
if (total_chunks_size + required_chunk_size <= max_total_size)
|
||||
{
|
||||
|
@ -8,10 +8,11 @@
|
||||
|
||||
#include "MemoryStatisticsOS.h"
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/getPageSize.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/ReadBufferFromMemory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -26,7 +27,6 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
static constexpr auto filename = "/proc/self/statm";
|
||||
static constexpr size_t PAGE_SIZE = 4096;
|
||||
|
||||
MemoryStatisticsOS::MemoryStatisticsOS()
|
||||
{
|
||||
@ -93,11 +93,12 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
|
||||
skipWhitespaceIfAny(in);
|
||||
readIntText(data.data_and_stack, in);
|
||||
|
||||
data.virt *= PAGE_SIZE;
|
||||
data.resident *= PAGE_SIZE;
|
||||
data.shared *= PAGE_SIZE;
|
||||
data.code *= PAGE_SIZE;
|
||||
data.data_and_stack *= PAGE_SIZE;
|
||||
size_t page_size = static_cast<size_t>(::getPageSize());
|
||||
data.virt *= page_size;
|
||||
data.resident *= page_size;
|
||||
data.shared *= page_size;
|
||||
data.code *= page_size;
|
||||
data.data_and_stack *= page_size;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/getPageSize.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/UTF8Helpers.h>
|
||||
@ -37,7 +38,7 @@ struct StringSearcherBase
|
||||
{
|
||||
#ifdef __SSE2__
|
||||
static constexpr auto n = sizeof(__m128i);
|
||||
const int page_size = getpagesize();
|
||||
const int page_size = ::getPageSize();
|
||||
|
||||
bool pageSafe(const void * const ptr) const
|
||||
{
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/formatReadable.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <common/getPageSize.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/MMapReadBufferFromFileDescriptor.h>
|
||||
|
||||
@ -38,7 +39,9 @@ void MMapReadBufferFromFileDescriptor::init(int fd_, size_t offset, size_t lengt
|
||||
ErrorCodes::CANNOT_ALLOCATE_MEMORY);
|
||||
|
||||
BufferBase::set(static_cast<char *>(buf), length, 0);
|
||||
ReadBuffer::padded = (length % 4096) > 0 && (length % 4096) <= (4096 - 15); /// TODO determine page size
|
||||
|
||||
size_t page_size = static_cast<size_t>(::getPageSize());
|
||||
ReadBuffer::padded = (length % page_size) > 0 && (length % page_size) <= (page_size - 15);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include <iostream>
|
||||
|
||||
#include <common/types.h>
|
||||
#include <common/getPageSize.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
@ -16,6 +17,7 @@ int main(int, char **)
|
||||
{
|
||||
static const size_t N = 100000;
|
||||
static const size_t BUF_SIZE = 1048576;
|
||||
size_t page_size = static_cast<size_t>(::getPageSize());
|
||||
|
||||
ReadBufferFromFile rand_in("/dev/urandom");
|
||||
unsigned rand = 0;
|
||||
@ -33,7 +35,7 @@ int main(int, char **)
|
||||
}
|
||||
|
||||
{
|
||||
ReadBufferFromFile rb("test1", BUF_SIZE, O_RDONLY | O_DIRECT, nullptr, 4096);
|
||||
ReadBufferFromFile rb("test1", BUF_SIZE, O_RDONLY | O_DIRECT, nullptr, page_size);
|
||||
String res;
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
readStringBinary(res, rb);
|
||||
@ -44,14 +46,14 @@ int main(int, char **)
|
||||
/// Write to file with O_DIRECT, read as usual.
|
||||
|
||||
{
|
||||
WriteBufferFromFile wb("test2", BUF_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096);
|
||||
WriteBufferFromFile wb("test2", BUF_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, page_size);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
writeStringBinary(test, wb);
|
||||
|
||||
if (wb.offset() % 4096 != 0)
|
||||
if (wb.offset() % page_size != 0)
|
||||
{
|
||||
size_t pad = 4096 - wb.offset() % 4096;
|
||||
size_t pad = page_size - wb.offset() % page_size;
|
||||
memset(wb.position(), 0, pad);
|
||||
wb.position() += pad;
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
#include <Common/TaskStatsInfoGetter.h>
|
||||
#include <Poco/File.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <common/getPageSize.h>
|
||||
#include <common/getThreadId.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <linux/taskstats.h>
|
||||
@ -61,8 +62,9 @@ static void do_io(size_t id)
|
||||
std::string path_dst = "test_out_" + std::to_string(id);
|
||||
|
||||
{
|
||||
size_t page_size = static_cast<size_t>(::getPageSize());
|
||||
ReadBufferFromFile rb("/dev/urandom");
|
||||
WriteBufferFromFile wb(path_dst, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096);
|
||||
WriteBufferFromFile wb(path_dst, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, page_size);
|
||||
copyData(rb, wb, copy_size);
|
||||
wb.close();
|
||||
}
|
||||
|
@ -40,8 +40,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
#define INDEX_BUFFER_SIZE 4096
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
@ -319,7 +317,7 @@ Pipe StorageStripeLog::read(
|
||||
return Pipe(std::make_shared<NullSource>(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID())));
|
||||
}
|
||||
|
||||
CompressedReadBufferFromFile index_in(disk->readFile(index_file, INDEX_BUFFER_SIZE));
|
||||
CompressedReadBufferFromFile index_in(disk->readFile(index_file, 4096));
|
||||
std::shared_ptr<const IndexForNativeFormat> index{std::make_shared<IndexForNativeFormat>(index_in, column_names_set)};
|
||||
|
||||
size_t size = index->blocks.size();
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <common/getPageSize.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iomanip>
|
||||
@ -46,7 +47,7 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
Memory<> direct_buf(block_size, sysconf(_SC_PAGESIZE));
|
||||
Memory<> direct_buf(block_size, ::getPageSize());
|
||||
std::vector<char> simple_buf(block_size);
|
||||
|
||||
char * buf;
|
||||
|
@ -14,6 +14,7 @@ int main(int, char **) { return 0; }
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <common/getPageSize.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
@ -52,7 +53,7 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block
|
||||
|
||||
std::vector<Memory<>> buffers(buffers_count);
|
||||
for (size_t i = 0; i < buffers_count; ++i)
|
||||
buffers[i] = Memory<>(block_size, sysconf(_SC_PAGESIZE));
|
||||
buffers[i] = Memory<>(block_size, ::getPageSize());
|
||||
|
||||
pcg64_fast rng(randomSeed());
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user