Merge pull request #11316 from ClickHouse/better-not-enough-memory-message

Better exception message in case when there is shortage of memory mappings

src/Common/Exception.cpp

@@ -5,8 +5,10 @@
 #include <Poco/String.h>
 #include <common/logger_useful.h>
 #include <IO/WriteHelpers.h>
+#include <IO/ReadHelpers.h>
 #include <IO/Operators.h>
 #include <IO/ReadBufferFromString.h>
+#include <IO/ReadBufferFromFile.h>
 #include <common/demangle.h>
 #include <Common/formatReadable.h>
 #include <Common/filesystemHelpers.h>
@@ -25,6 +27,8 @@ namespace ErrorCodes
     extern const int STD_EXCEPTION;
     extern const int UNKNOWN_EXCEPTION;
     extern const int LOGICAL_ERROR;
+    extern const int CANNOT_ALLOCATE_MEMORY;
+    extern const int CANNOT_MREMAP;
 }
 
 
@@ -156,6 +160,64 @@ static void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string &
 #endif
 }
 
+
+/** It is possible that the system has enough memory,
+  *  but we have shortage of the number of available memory mappings.
+  * Provide good diagnostic to user in that case.
+  */
+static void getNotEnoughMemoryMessage(std::string & msg)
+{
+#if defined(__linux__)
+    try
+    {
+        static constexpr size_t buf_size = 4096;
+        char buf[buf_size];
+
+        UInt64 max_map_count = 0;
+        {
+            ReadBufferFromFile file("/proc/sys/vm/max_map_count", buf_size, -1, buf);
+            readText(max_map_count, file);
+        }
+
+        UInt64 num_maps = 0;
+        {
+            ReadBufferFromFile file("/proc/self/maps", buf_size, -1, buf);
+            while (!file.eof())
+            {
+                char * next_pos = find_first_symbols<'\n'>(file.position(), file.buffer().end());
+                file.position() = next_pos;
+
+                if (!file.hasPendingData())
+                    continue;
+
+                if (*file.position() == '\n')
+                {
+                    ++num_maps;
+                    ++file.position();
+                }
+            }
+        }
+
+        if (num_maps > max_map_count * 0.99)
+        {
+            msg += fmt::format(
+                "\nIt looks like that the process is near the limit on number of virtual memory mappings."
+                "\nCurrent number of mappings (/proc/self/maps): {}."
+                "\nLimit on number of mappings (/proc/sys/vm/max_map_count): {}."
+                "\nYou should increase the limit for vm.max_map_count in /etc/sysctl.conf"
+                "\n",
+                num_maps, max_map_count);
+        }
+    }
+    catch (...)
+    {
+        msg += "\nCannot obtain additional info about memory usage.";
+    }
+#else
+    (void)msg;
+#endif
+}
+
 static std::string getExtraExceptionInfo(const std::exception & e)
 {
     String msg;
@@ -170,6 +232,13 @@ static std::string getExtraExceptionInfo(const std::exception & e)
         {
             if (errno_exception->getErrno() == ENOSPC && errno_exception->getPath())
                 getNoSpaceLeftInfoMessage(errno_exception->getPath().value(), msg);
+            else if (errno_exception->code() == ErrorCodes::CANNOT_ALLOCATE_MEMORY
+                || errno_exception->code() == ErrorCodes::CANNOT_MREMAP)
+                getNotEnoughMemoryMessage(msg);
+        }
+        else if (dynamic_cast<const std::bad_alloc *>(&e))
+        {
+            getNotEnoughMemoryMessage(msg);
         }
     }
     catch (...)

src/Functions/trap.cpp

@@ -7,11 +7,13 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <Columns/ColumnString.h>
 #include <Interpreters/Context.h>
+#include <ext/scope_guard.h>
 
 #include <thread>
 #include <memory>
 #include <cstdlib>
 #include <unistd.h>
+#include <sys/mman.h>
 
 
 namespace DB
@@ -22,6 +24,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_COLUMN;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int BAD_ARGUMENTS;
+    extern const int CANNOT_ALLOCATE_MEMORY;
 }
 
 
@@ -132,6 +135,25 @@ public:
             {
                 (void)context.getCurrentQueryId();
             }
+            else if (mode == "mmap many")
+            {
+                std::vector<void *> maps;
+                SCOPE_EXIT(
+                {
+                    //for (void * map : maps)
+                    //    munmap(map, 4096);
+                });
+
+                while (true)
+                {
+                    void * hint = reinterpret_cast<void *>(
+                        std::uniform_int_distribution<intptr_t>(0x100000000000UL, 0x700000000000UL)(thread_local_rng));
+                    void * map = mmap(hint, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                    if (MAP_FAILED == map)
+                        throwFromErrno("Allocator: Cannot mmap", ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+                    maps.push_back(map);
+                }
+            }
             else
                 throw Exception("Unknown trap mode", ErrorCodes::BAD_ARGUMENTS);
         }