diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp new file mode 100644 index 00000000000..adc5bf5d904 --- /dev/null +++ b/src/Common/MemoryStatisticsOS.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include + +#include "MemoryStatisticsOS.h" + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; +} + +static constexpr auto filename = "/proc/self/statm"; +static constexpr size_t PAGE_SIZE = 4096; + +MemoryStatisticsOS::MemoryStatisticsOS() +{ + fd = ::open(filename, O_RDONLY | O_CLOEXEC); + + if (-1 == fd) + throwFromErrno("Cannot open file " + std::string(filename), errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); +} + +MemoryStatisticsOS::~MemoryStatisticsOS() +{ + if (0 != ::close(fd)) + tryLogCurrentException(__PRETTY_FUNCTION__); +} + +MemoryStatisticsOS::Data MemoryStatisticsOS::get() const +{ + Data data; + + constexpr size_t buf_size = 1024; + char buf[buf_size]; + + ssize_t res = 0; + + do + { + res = ::pread(fd, buf, buf_size, 0); + + if (-1 == res) + { + if (errno == EINTR) + continue; + + throwFromErrno("Cannot read from file " + std::string(filename), ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); + } + + assert(res >= 0); + break; + } while (true); + + ReadBufferFromMemory in(buf, res); + + uint64_t unused; + readIntText(data.virt, in); + skipWhitespaceIfAny(in); + readIntText(data.resident, in); + skipWhitespaceIfAny(in); + readIntText(data.shared, in); + skipWhitespaceIfAny(in); + readIntText(data.code, in); + skipWhitespaceIfAny(in); + readIntText(unused, in); + skipWhitespaceIfAny(in); + readIntText(data.data_and_stack, in); + + data.virt *= PAGE_SIZE; + data.resident *= PAGE_SIZE; + data.shared *= PAGE_SIZE; + data.code *= PAGE_SIZE; + data.data_and_stack *= PAGE_SIZE; + + return data; +} + +} diff --git a/src/Common/MemoryStatisticsOS.h b/src/Common/MemoryStatisticsOS.h new file mode 100644 index 00000000000..97caf4e8fbe --- /dev/null +++ b/src/Common/MemoryStatisticsOS.h @@ -0,0 +1,40 @@ +#pragma once +#include + + +namespace DB +{ + +/** Opens a file /proc/self/mstat. Keeps it open and reads memory statistics via 'pread'. + * This is Linux specific. + * See: man procfs + * + * Note: a class is used instead of a single function to avoid excessive file open/close on every use. + * pread is used to avoid lseek. + * + * Actual performance is from 1 to 5 million iterations per second. + */ +class MemoryStatisticsOS +{ +public: + /// In number of bytes. + struct Data + { + uint64_t virt; + uint64_t resident; + uint64_t shared; + uint64_t code; + uint64_t data_and_stack; + }; + + MemoryStatisticsOS(); + ~MemoryStatisticsOS(); + + /// Thread-safe. + Data get() const; + +private: + int fd; +}; + +} diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 4ed21ba4ef2..2ddbee43f5f 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -68,3 +68,6 @@ target_link_libraries (symbol_index PRIVATE clickhouse_common_io) add_executable (chaos_sanitizer chaos_sanitizer.cpp) target_link_libraries (chaos_sanitizer PRIVATE clickhouse_common_io) + +add_executable (memory_statistics_os_perf memory_statistics_os_perf.cpp) +target_link_libraries (memory_statistics_os_perf PRIVATE clickhouse_common_io) diff --git a/src/Common/tests/memory_statistics_os_perf.cpp b/src/Common/tests/memory_statistics_os_perf.cpp new file mode 100644 index 00000000000..dacd959fe1d --- /dev/null +++ b/src/Common/tests/memory_statistics_os_perf.cpp @@ -0,0 +1,23 @@ +#include +#include + + +int main(int argc, char ** argv) +{ + using namespace DB; + + size_t num_iterations = argc >= 2 ? std::stoull(argv[1]) : 1000000; + MemoryStatisticsOS stats; + + uint64_t counter = 0; + for (size_t i = 0; i < num_iterations; ++i) + { + MemoryStatisticsOS::Data data = stats.get(); + counter += data.resident; + } + + std::cerr << (counter / num_iterations) << '\n'; + return 0; +} + + diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 304db048f4f..b635c726b4c 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -12,6 +12,7 @@ #include #include + #if !defined(ARCADIA_BUILD) # include "config_core.h" #endif @@ -130,6 +131,19 @@ void AsynchronousMetrics::update() set("Uptime", context.getUptimeSeconds()); + /// Process memory usage according to OS +#if defined(OS_LINUX) + { + MemoryStatisticsOS::Data data = memory_stat.get(); + + set("MemoryVirtual", data.virt); + set("MemoryResident", data.resident); + set("MemoryShared", data.shared); + set("MemoryCode", data.code); + set("MemoryDataAndStack", data.data_and_stack); + } +#endif + { auto databases = DatabaseCatalog::instance().getDatabases(); diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 8ccefb9e930..b5e1765cd25 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -44,6 +45,7 @@ private: Container container; mutable std::mutex container_mutex; + MemoryStatisticsOS memory_stat; ThreadFromGlobalPool thread; void run(); diff --git a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 9e0737810df..ed5eb5b4d2e 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -27,6 +27,8 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( message << "Reading " << data_part->getMarksCount() << " marks from part " << data_part->name << ", total " << data_part->rows_count << " rows starting from the beginning of the part"; + if (columns_to_read.size() == 1) /// Print column name but don't pollute logs in case of many columns. + message << ", column " << columns_to_read.front(); LOG_TRACE(log, message.rdbuf()); }