Merge pull request #10361 from ClickHouse/async-metric-memory-usage

Add memory usage from OS to system.asynchronous_metrics.

src/Common/MemoryStatisticsOS.cpp

@@ -0,0 +1,90 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <cassert>
+
+#include "MemoryStatisticsOS.h"
+
+#include <Common/Exception.h>
+#include <IO/ReadBufferFromMemory.h>
+#include <IO/ReadHelpers.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int FILE_DOESNT_EXIST;
+    extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
+}
+
+static constexpr auto filename = "/proc/self/statm";
+static constexpr size_t PAGE_SIZE = 4096;
+
+MemoryStatisticsOS::MemoryStatisticsOS()
+{
+    fd = ::open(filename, O_RDONLY | O_CLOEXEC);
+
+    if (-1 == fd)
+        throwFromErrno("Cannot open file " + std::string(filename), errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
+}
+
+MemoryStatisticsOS::~MemoryStatisticsOS()
+{
+    if (0 != ::close(fd))
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+}
+
+MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
+{
+    Data data;
+
+    constexpr size_t buf_size = 1024;
+    char buf[buf_size];
+
+    ssize_t res = 0;
+
+    do
+    {
+        res = ::pread(fd, buf, buf_size, 0);
+
+        if (-1 == res)
+        {
+            if (errno == EINTR)
+                continue;
+
+            throwFromErrno("Cannot read from file " + std::string(filename), ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
+        }
+
+        assert(res >= 0);
+        break;
+    } while (true);
+
+    ReadBufferFromMemory in(buf, res);
+
+    uint64_t unused;
+    readIntText(data.virt, in);
+    skipWhitespaceIfAny(in);
+    readIntText(data.resident, in);
+    skipWhitespaceIfAny(in);
+    readIntText(data.shared, in);
+    skipWhitespaceIfAny(in);
+    readIntText(data.code, in);
+    skipWhitespaceIfAny(in);
+    readIntText(unused, in);
+    skipWhitespaceIfAny(in);
+    readIntText(data.data_and_stack, in);
+
+    data.virt *= PAGE_SIZE;
+    data.resident *= PAGE_SIZE;
+    data.shared *= PAGE_SIZE;
+    data.code *= PAGE_SIZE;
+    data.data_and_stack *= PAGE_SIZE;
+
+    return data;
+}
+
+}

src/Common/MemoryStatisticsOS.h

@@ -0,0 +1,40 @@
+#pragma once
+#include <cstdint>
+
+
+namespace DB
+{
+
+/** Opens a file /proc/self/mstat. Keeps it open and reads memory statistics via 'pread'.
+  * This is Linux specific.
+  * See: man procfs
+  *
+  * Note: a class is used instead of a single function to avoid excessive file open/close on every use.
+  * pread is used to avoid lseek.
+  *
+  * Actual performance is from 1 to 5 million iterations per second.
+  */
+class MemoryStatisticsOS
+{
+public:
+    /// In number of bytes.
+    struct Data
+    {
+        uint64_t virt;
+        uint64_t resident;
+        uint64_t shared;
+        uint64_t code;
+        uint64_t data_and_stack;
+    };
+
+    MemoryStatisticsOS();
+    ~MemoryStatisticsOS();
+
+    /// Thread-safe.
+    Data get() const;
+
+private:
+    int fd;
+};
+
+}

src/Common/tests/CMakeLists.txt

@@ -68,3 +68,6 @@ target_link_libraries (symbol_index PRIVATE clickhouse_common_io)
 
 add_executable (chaos_sanitizer chaos_sanitizer.cpp)
 target_link_libraries (chaos_sanitizer PRIVATE clickhouse_common_io)
+
+add_executable (memory_statistics_os_perf memory_statistics_os_perf.cpp)
+target_link_libraries (memory_statistics_os_perf PRIVATE clickhouse_common_io)

src/Common/tests/memory_statistics_os_perf.cpp

@@ -0,0 +1,23 @@
+#include <Common/MemoryStatisticsOS.h>
+#include <iostream>
+
+
+int main(int argc, char ** argv)
+{
+    using namespace DB;
+
+    size_t num_iterations = argc >= 2 ? std::stoull(argv[1]) : 1000000;
+    MemoryStatisticsOS stats;
+
+    uint64_t counter = 0;
+    for (size_t i = 0; i < num_iterations; ++i)
+    {
+        MemoryStatisticsOS::Data data = stats.get();
+        counter += data.resident;
+    }
+
+    std::cerr << (counter / num_iterations) << '\n';
+    return 0;
+}
+
+

src/Interpreters/AsynchronousMetrics.cpp

@@ -12,6 +12,7 @@
 #include <Databases/IDatabase.h>
 #include <chrono>
 
+
 #if !defined(ARCADIA_BUILD)
 #    include "config_core.h"
 #endif
@@ -130,6 +131,19 @@ void AsynchronousMetrics::update()
 
     set("Uptime", context.getUptimeSeconds());
 
+    /// Process memory usage according to OS
+#if defined(OS_LINUX)
+    {
+        MemoryStatisticsOS::Data data = memory_stat.get();
+
+        set("MemoryVirtual", data.virt);
+        set("MemoryResident", data.resident);
+        set("MemoryShared", data.shared);
+        set("MemoryCode", data.code);
+        set("MemoryDataAndStack", data.data_and_stack);
+    }
+#endif
+
     {
         auto databases = DatabaseCatalog::instance().getDatabases();
 

src/Interpreters/AsynchronousMetrics.h

@@ -6,6 +6,7 @@
 #include <unordered_map>
 #include <string>
 #include <Common/ThreadPool.h>
+#include <Common/MemoryStatisticsOS.h>
 
 
 namespace DB
@@ -44,6 +45,7 @@ private:
     Container container;
     mutable std::mutex container_mutex;
 
+    MemoryStatisticsOS memory_stat;
     ThreadFromGlobalPool thread;
 
     void run();

src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp

@@ -27,6 +27,8 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream(
         message << "Reading " << data_part->getMarksCount() << " marks from part " << data_part->name
             << ", total " << data_part->rows_count
             << " rows starting from the beginning of the part";
+        if (columns_to_read.size() == 1)    /// Print column name but don't pollute logs in case of many columns.
+            message << ", column " << columns_to_read.front();
 
         LOG_TRACE(log, message.rdbuf());
     }