Merge pull request #64365 from ClickHouse/fix-gwp-asan

Try to fix GWPAsan

CMakeLists.txt

@@ -399,7 +399,7 @@ option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
 # but GWP-ASan also wants to use mmap frequently,
 # and due to a large number of memory mappings,
 # it does not work together well.
-if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
+if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") OR SANITIZE)
     set(ENABLE_GWP_ASAN OFF)
 endif ()
 

programs/server/Server.cpp

@@ -1671,6 +1671,10 @@ try
             if (global_context->isServerCompletelyStarted())
                 CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
 
+#if USE_GWP_ASAN
+            GWPAsan::setForceSampleProbability(new_server_settings.gwp_asan_force_sample_probability);
+#endif
+
             ProfileEvents::increment(ProfileEvents::MainConfigLoads);
 
             /// Must be the last.
@@ -2120,6 +2124,10 @@ try
 
         CannotAllocateThreadFaultInjector::setFaultProbability(server_settings.cannot_allocate_thread_fault_injection_probability);
 
+#if USE_GWP_ASAN
+        GWPAsan::setForceSampleProbability(server_settings.gwp_asan_force_sample_probability);
+#endif
+
         try
         {
             global_context->startClusterDiscovery();

src/CMakeLists.txt

@@ -285,7 +285,7 @@ if (TARGET ch_contrib::llvm)
 endif ()
 
 if (TARGET ch_contrib::gwp_asan)
-    target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::gwp_asan)
+    target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::gwp_asan)
     target_link_libraries (clickhouse_new_delete PRIVATE ch_contrib::gwp_asan)
 endif()
 

src/Common/Allocator.cpp

@@ -1,8 +1,9 @@
 #include <Common/Allocator.h>
-#include <Common/Exception.h>
-#include <Common/logger_useful.h>
-#include <Common/formatReadable.h>
 #include <Common/CurrentMemoryTracker.h>
+#include <Common/Exception.h>
+#include <Common/GWPAsan.h>
+#include <Common/formatReadable.h>
+#include <Common/logger_useful.h>
 
 #include <base/errnoToString.h>
 #include <base/getPageSize.h>
@@ -10,6 +11,12 @@
 #include <Poco/Logger.h>
 #include <sys/mman.h> /// MADV_POPULATE_WRITE
 
+namespace ProfileEvents
+{
+    extern const Event GWPAsanAllocateSuccess;
+    extern const Event GWPAsanAllocateFailed;
+    extern const Event GWPAsanFree;
+}
 
 namespace DB
 {
@@ -60,6 +67,27 @@ template <bool clear_memory, bool populate>
 void * allocNoTrack(size_t size, size_t alignment)
 {
     void * buf;
+#if USE_GWP_ASAN
+    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    {
+        if (void * ptr = GWPAsan::GuardedAlloc.allocate(size, alignment))
+        {
+            if constexpr (clear_memory)
+                memset(ptr, 0, size);
+
+            if constexpr (populate)
+                prefaultPages(ptr, size);
+
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
+
+            return ptr;
+        }
+        else
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+        }
+    }
+#endif
     if (alignment <= MALLOC_MIN_ALIGNMENT)
     {
         if constexpr (clear_memory)
@@ -91,6 +119,15 @@ void * allocNoTrack(size_t size, size_t alignment)
 
 void freeNoTrack(void * buf)
 {
+#if USE_GWP_ASAN
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(buf)))
+    {
+        ProfileEvents::increment(ProfileEvents::GWPAsanFree);
+        GWPAsan::GuardedAlloc.deallocate(buf);
+        return;
+    }
+#endif
+
     ::free(buf);
 }
 
@@ -144,8 +181,54 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
     {
         /// nothing to do.
         /// BTW, it's not possible to change alignment while doing realloc.
+        return buf;
     }
-    else if (alignment <= MALLOC_MIN_ALIGNMENT)
+
+#if USE_GWP_ASAN
+    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    {
+        if (void * ptr = GWPAsan::GuardedAlloc.allocate(new_size, alignment))
+        {
+            auto trace_free = CurrentMemoryTracker::free(old_size);
+            auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
+            trace_free.onFree(buf, old_size);
+
+            memcpy(ptr, buf, std::min(old_size, new_size));
+            free(buf, old_size);
+            trace_alloc.onAlloc(buf, new_size);
+
+            if constexpr (clear_memory)
+                if (new_size > old_size)
+                    memset(reinterpret_cast<char *>(ptr) + old_size, 0, new_size - old_size);
+
+            if constexpr (populate)
+                prefaultPages(ptr, new_size);
+
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
+            return ptr;
+        }
+        else
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+        }
+    }
+
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(buf)))
+    {
+        /// Big allocs that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods.
+        void * new_buf = alloc(new_size, alignment);
+        memcpy(new_buf, buf, std::min(old_size, new_size));
+        free(buf, old_size);
+        buf = new_buf;
+
+        if constexpr (populate)
+            prefaultPages(buf, new_size);
+
+        return buf;
+    }
+#endif
+
+    if (alignment <= MALLOC_MIN_ALIGNMENT)
     {
         /// Resize malloc'd memory region with no special alignment requirement.
         auto trace_free = CurrentMemoryTracker::free(old_size);

src/Common/GWPAsan.cpp

@@ -0,0 +1,226 @@
+#include <Common/GWPAsan.h>
+
+#if USE_GWP_ASAN
+#    include <IO/ReadHelpers.h>
+#    include <gwp_asan/common.h>
+#    include <gwp_asan/crash_handler.h>
+#    include <gwp_asan/guarded_pool_allocator.h>
+#    include <gwp_asan/optional/options_parser.h>
+#    include <Common/ErrorCodes.h>
+#    include <Common/Exception.h>
+#    include <Common/Logger.h>
+#    include <Common/StackTrace.h>
+#    include <Common/logger_useful.h>
+
+#    include <atomic>
+#    include <iostream>
+
+namespace GWPAsan
+{
+
+namespace
+{
+size_t getBackTrace(uintptr_t * trace_buffer, size_t buffer_size)
+{
+    StackTrace stacktrace;
+    auto trace_size = std::min(buffer_size, stacktrace.getSize());
+    const auto & frame_pointers = stacktrace.getFramePointers();
+    memcpy(trace_buffer, frame_pointers.data(), trace_size * sizeof(uintptr_t));
+    return trace_size;
+}
+
+__attribute__((__format__ (__printf__, 1, 0)))
+void printString(const char * format, ...) // NOLINT(cert-dcl50-cpp)
+{
+    std::array<char, 1024> formatted;
+    va_list args;
+    va_start(args, format);
+
+    if (vsnprintf(formatted.data(), formatted.size(), format, args) > 0)
+        std::cerr << formatted.data() << std::endl;
+
+    va_end(args);
+}
+
+}
+
+gwp_asan::GuardedPoolAllocator GuardedAlloc;
+
+static bool guarded_alloc_initialized = []
+{
+    const char * env_options_raw = std::getenv("GWP_ASAN_OPTIONS"); // NOLINT(concurrency-mt-unsafe)
+    if (env_options_raw)
+        gwp_asan::options::initOptions(env_options_raw, printString);
+
+    auto & opts = gwp_asan::options::getOptions();
+    if (!env_options_raw || !std::string_view{env_options_raw}.contains("MaxSimultaneousAllocations"))
+        opts.MaxSimultaneousAllocations = 1024;
+
+    if (!env_options_raw || !std::string_view{env_options_raw}.contains("SampleRate"))
+        opts.SampleRate = 50000;
+
+    opts.Backtrace = getBackTrace;
+    GuardedAlloc.init(opts);
+
+    return true;
+}();
+
+bool isGWPAsanError(uintptr_t fault_address)
+{
+    const auto * state = GuardedAlloc.getAllocatorState();
+    if (state->FailureType != gwp_asan::Error::UNKNOWN && state->FailureAddress != 0)
+        return true;
+
+    return fault_address < state->GuardedPagePoolEnd && state->GuardedPagePool <= fault_address;
+}
+
+namespace
+{
+
+struct ScopedEndOfReportDecorator
+{
+    explicit ScopedEndOfReportDecorator(Poco::LoggerPtr log_) : log(std::move(log_)) { }
+    ~ScopedEndOfReportDecorator() { LOG_FATAL(log, "*** End GWP-ASan report ***"); }
+    Poco::LoggerPtr log;
+};
+
+// Prints the provided error and metadata information.
+void printHeader(gwp_asan::Error error, uintptr_t fault_address, const gwp_asan::AllocationMetadata * allocation_meta, Poco::LoggerPtr log)
+{
+    bool access_was_in_bounds = false;
+    std::string description;
+    if (error != gwp_asan::Error::UNKNOWN && allocation_meta != nullptr)
+    {
+        uintptr_t address = __gwp_asan_get_allocation_address(allocation_meta);
+        size_t size = __gwp_asan_get_allocation_size(allocation_meta);
+        if (fault_address < address)
+        {
+            description = fmt::format(
+                "({} byte{} to the left of a {}-byte allocation at 0x{}) ",
+                address - fault_address,
+                (address - fault_address == 1) ? "" : "s",
+                size,
+                address);
+        }
+        else if (fault_address > address)
+        {
+            description = fmt::format(
+                "({} byte{} to the right of a {}-byte allocation at 0x{}) ",
+                fault_address - address,
+                (fault_address - address == 1) ? "" : "s",
+                size,
+                address);
+        }
+        else if (error == gwp_asan::Error::DOUBLE_FREE)
+        {
+            description = fmt::format("(a {}-byte allocation) ", size);
+        }
+        else
+        {
+            access_was_in_bounds = true;
+            description = fmt::format(
+                "({} byte{} into a {}-byte allocation at 0x{}) ",
+                fault_address - address,
+                (fault_address - address == 1) ? "" : "s",
+                size,
+                address);
+        }
+    }
+
+    uint64_t thread_id = gwp_asan::getThreadID();
+    std::string thread_id_string = thread_id == gwp_asan::kInvalidThreadID ? "<unknown" : fmt::format("{}", thread_id);
+
+    std::string_view out_of_bounds_and_use_after_free_warning;
+    if (error == gwp_asan::Error::USE_AFTER_FREE && !access_was_in_bounds)
+    {
+        out_of_bounds_and_use_after_free_warning = " (warning: buffer overflow/underflow detected on a free()'d "
+                                                   "allocation. This either means you have a buffer-overflow and a "
+                                                   "use-after-free at the same time, or you have a long-lived "
+                                                   "use-after-free bug where the allocation/deallocation metadata below "
+                                                   "has already been overwritten and is likely bogus)";
+    }
+
+    LOG_FATAL(
+     log,
+        "{}{} at 0x{} {}by thread {} here:",
+        gwp_asan::ErrorToString(error),
+        out_of_bounds_and_use_after_free_warning,
+        fault_address,
+        description,
+        thread_id_string);
+}
+
+}
+
+void printReport([[maybe_unused]] uintptr_t fault_address)
+{
+    const auto logger = getLogger("GWPAsan");
+    const auto * state = GuardedAlloc.getAllocatorState();
+    if (uintptr_t internal_error_ptr = __gwp_asan_get_internal_crash_address(state); internal_error_ptr)
+        fault_address = internal_error_ptr;
+
+    const gwp_asan::AllocationMetadata * allocation_meta = __gwp_asan_get_metadata(state, GuardedAlloc.getMetadataRegion(), fault_address);
+
+    static constexpr std::string_view unknown_crash_text =
+        "GWP-ASan cannot provide any more information about this error. This may "
+        "occur due to a wild memory access into the GWP-ASan pool, or an "
+        "overflow/underflow that is > 512B in length.\n";
+
+    if (allocation_meta == nullptr)
+    {
+        LOG_FATAL(logger, "*** GWP-ASan detected a memory error ***");
+        ScopedEndOfReportDecorator decorator(logger);
+        LOG_FATAL(logger, fmt::runtime(unknown_crash_text));
+        return;
+    }
+
+    LOG_FATAL(logger, "*** GWP-ASan detected a memory error ***");
+    ScopedEndOfReportDecorator decorator(logger);
+
+    gwp_asan::Error error = __gwp_asan_diagnose_error(state, allocation_meta, fault_address);
+    if (error == gwp_asan::Error::UNKNOWN)
+    {
+        LOG_FATAL(logger, fmt::runtime(unknown_crash_text));
+        return;
+    }
+
+    // Print the error header.
+    printHeader(error, fault_address, allocation_meta, logger);
+
+    static constexpr size_t maximum_stack_frames = 512;
+    std::array<uintptr_t, maximum_stack_frames> trace;
+
+    // Maybe print the deallocation trace.
+    if (__gwp_asan_is_deallocated(allocation_meta))
+    {
+        uint64_t thread_id = __gwp_asan_get_deallocation_thread_id(allocation_meta);
+        if (thread_id == gwp_asan::kInvalidThreadID)
+            LOG_FATAL(logger, "0x{} was deallocated by thread <unknown> here:", fault_address);
+        else
+            LOG_FATAL(logger, "0x{} was deallocated by thread {} here:", fault_address, thread_id);
+        const auto trace_length = __gwp_asan_get_deallocation_trace(allocation_meta, trace.data(), maximum_stack_frames);
+        StackTrace::toStringEveryLine(
+            reinterpret_cast<void **>(trace.data()), 0, trace_length, [&](const auto line) { LOG_FATAL(logger, fmt::runtime(line)); });
+    }
+
+    // Print the allocation trace.
+    uint64_t thread_id = __gwp_asan_get_allocation_thread_id(allocation_meta);
+    if (thread_id == gwp_asan::kInvalidThreadID)
+        LOG_FATAL(logger, "0x{} was allocated by thread <unknown> here:", fault_address);
+    else
+        LOG_FATAL(logger, "0x{} was allocated by thread {} here:", fault_address, thread_id);
+    const auto trace_length = __gwp_asan_get_allocation_trace(allocation_meta, trace.data(), maximum_stack_frames);
+    StackTrace::toStringEveryLine(
+        reinterpret_cast<void **>(trace.data()), 0, trace_length, [&](const auto line) { LOG_FATAL(logger, fmt::runtime(line)); });
+}
+
+std::atomic<double> force_sample_probability = 0.0;
+
+void setForceSampleProbability(double value)
+{
+    force_sample_probability.store(value, std::memory_order_relaxed);
+}
+
+}
+
+#endif

src/Common/GWPAsan.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_GWP_ASAN
+
+#include <gwp_asan/guarded_pool_allocator.h>
+#include <Common/thread_local_rng.h>
+
+#include <atomic>
+#include <random>
+
+namespace GWPAsan
+{
+
+extern gwp_asan::GuardedPoolAllocator GuardedAlloc;
+
+bool isGWPAsanError(uintptr_t fault_address);
+
+void printReport(uintptr_t fault_address);
+
+extern std::atomic<double> force_sample_probability;
+
+void setForceSampleProbability(double value);
+
+inline bool shouldForceSample()
+{
+    std::bernoulli_distribution dist(force_sample_probability.load(std::memory_order_relaxed));
+    return dist(thread_local_rng);
+}
+
+}
+
+#endif

src/Common/PODArray.h

@@ -1,17 +1,20 @@
 #pragma once
 
-#include <Common/Allocator.h>
-#include <Common/BitHelpers.h>
-#include <Common/memcpySmall.h>
-#include <Common/PODArray_fwd.h>
+#include "config.h"
+
 #include <base/getPageSize.h>
 #include <boost/noncopyable.hpp>
+#include <Common/Allocator.h>
+#include <Common/BitHelpers.h>
+#include <Common/GWPAsan.h>
+#include <Common/PODArray_fwd.h>
+#include <Common/memcpySmall.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
 #include <cstdlib>
 #include <cstring>
-#include <cstddef>
-#include <cassert>
-#include <algorithm>
-#include <memory>
 
 #ifndef NDEBUG
 #include <sys/mman.h>
@@ -112,6 +115,11 @@ protected:
     template <typename ... TAllocatorParams>
     void alloc(size_t bytes, TAllocatorParams &&... allocator_params)
     {
+#if USE_GWP_ASAN
+        if (unlikely(GWPAsan::shouldForceSample()))
+            gwp_asan::getThreadLocals()->NextSampleCounter = 1;
+#endif
+
         char * allocated = reinterpret_cast<char *>(TAllocator::alloc(bytes, std::forward<TAllocatorParams>(allocator_params)...));
 
         c_start = allocated + pad_left;
@@ -141,6 +149,11 @@ protected:
             return;
         }
 
+#if USE_GWP_ASAN
+        if (unlikely(GWPAsan::shouldForceSample()))
+            gwp_asan::getThreadLocals()->NextSampleCounter = 1;
+#endif
+
         unprotect();
 
         ptrdiff_t end_diff = c_end - c_start;

src/Common/ProfileEvents.cpp

@@ -754,6 +754,10 @@ The server successfully detected this situation and will download merged part fr
     \
     M(ReadWriteBufferFromHTTPRequestsSent, "Number of HTTP requests sent by ReadWriteBufferFromHTTP") \
     M(ReadWriteBufferFromHTTPBytes, "Total size of payload bytes received and sent by ReadWriteBufferFromHTTP. Doesn't include HTTP headers.") \
+    \
+    M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan") \
+    M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)") \
+    M(GWPAsanFree, "Number of free operations done by GWPAsan") \
 
 
 #ifdef APPLY_FOR_EXTERNAL_EVENTS

src/Common/memory.h

@@ -5,6 +5,8 @@
 
 #include <Common/Concepts.h>
 #include <Common/CurrentMemoryTracker.h>
+#include <Common/ProfileEvents.h>
+#include <Common/GWPAsan.h>
 #include "config.h"
 
 #if USE_JEMALLOC
@@ -15,11 +17,12 @@
 #    include <cstdlib>
 #endif
 
-#if USE_GWP_ASAN
-#    include <gwp_asan/guarded_pool_allocator.h>
-
-static gwp_asan::GuardedPoolAllocator GuardedAlloc;
-#endif
+namespace ProfileEvents
+{
+    extern const Event GWPAsanAllocateSuccess;
+    extern const Event GWPAsanAllocateFailed;
+    extern const Event GWPAsanFree;
+}
 
 namespace Memory
 {
@@ -34,17 +37,31 @@ requires DB::OptionalArgument<TAlign...>
 inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align)
 {
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
     {
         if constexpr (sizeof...(TAlign) == 1)
         {
-            if (void * ptr = GuardedAlloc.allocate(size, alignToSizeT(align...)))
+            if (void * ptr = GWPAsan::GuardedAlloc.allocate(size, alignToSizeT(align...)))
+            {
+                ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
                 return ptr;
+            }
+            else
+            {
+                ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+            }
         }
         else
         {
-            if (void * ptr = GuardedAlloc.allocate(size))
+            if (void * ptr = GWPAsan::GuardedAlloc.allocate(size))
+            {
+                ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
                 return ptr;
+            }
+            else
+            {
+                ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+            }
 
         }
     }
@@ -66,10 +83,17 @@ inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align)
 inline ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
     {
-        if (void * ptr = GuardedAlloc.allocate(size))
+        if (void * ptr = GWPAsan::GuardedAlloc.allocate(size))
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
             return ptr;
+        }
+        else
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+        }
     }
 #endif
     return malloc(size);
@@ -78,10 +102,17 @@ inline ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
 inline ALWAYS_INLINE void * newNoExept(std::size_t size, std::align_val_t align) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
     {
-        if (void * ptr = GuardedAlloc.allocate(size, alignToSizeT(align)))
+        if (void * ptr = GWPAsan::GuardedAlloc.allocate(size, alignToSizeT(align)))
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateSuccess);
             return ptr;
+        }
+        else
+        {
+            ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
+        }
     }
 #endif
     return aligned_alloc(static_cast<size_t>(align), size);
@@ -90,9 +121,10 @@ inline ALWAYS_INLINE void * newNoExept(std::size_t size, std::align_val_t align)
 inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.pointerIsMine(ptr)))
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(ptr)))
     {
-        GuardedAlloc.deallocate(ptr);
+        ProfileEvents::increment(ProfileEvents::GWPAsanFree);
+        GWPAsan::GuardedAlloc.deallocate(ptr);
         return;
     }
 #endif
@@ -109,9 +141,10 @@ inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size, TAlign... al
         return;
 
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.pointerIsMine(ptr)))
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(ptr)))
     {
-        GuardedAlloc.deallocate(ptr);
+        ProfileEvents::increment(ProfileEvents::GWPAsanFree);
+        GWPAsan::GuardedAlloc.deallocate(ptr);
         return;
     }
 #endif
@@ -129,9 +162,10 @@ requires DB::OptionalArgument<TAlign...>
 inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]], TAlign... /* align */) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.pointerIsMine(ptr)))
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(ptr)))
     {
-        GuardedAlloc.deallocate(ptr);
+        ProfileEvents::increment(ProfileEvents::GWPAsanFree);
+        GWPAsan::GuardedAlloc.deallocate(ptr);
         return;
     }
 #endif
@@ -183,10 +217,10 @@ inline ALWAYS_INLINE size_t untrackMemory(void * ptr [[maybe_unused]], Allocatio
     std::size_t actual_size = 0;
 
 #if USE_GWP_ASAN
-    if (unlikely(GuardedAlloc.pointerIsMine(ptr)))
+    if (unlikely(GWPAsan::GuardedAlloc.pointerIsMine(ptr)))
     {
         if (!size)
-            size = GuardedAlloc.getSize(ptr);
+            size = GWPAsan::GuardedAlloc.getSize(ptr);
         trace = CurrentMemoryTracker::free(size);
         return size;
     }

src/Common/new_delete.cpp

@@ -1,5 +1,4 @@
 #include <cassert>
-#include <iostream>
 #include <new>
 #include "config.h"
 #include <Common/memory.h>
@@ -42,27 +41,6 @@ static struct InitializeJemallocZoneAllocatorForOSX
 } initializeJemallocZoneAllocatorForOSX;
 #endif
 
-#if USE_GWP_ASAN
-
-#include <gwp_asan/optional/options_parser.h>
-
-/// Both clickhouse_new_delete and clickhouse_common_io links gwp_asan, but It should only init once, otherwise it
-/// will cause unexpected deadlock.
-static struct InitGwpAsan
-{
-    InitGwpAsan()
-    {
-         gwp_asan::options::initOptions();
-         gwp_asan::options::Options &opts = gwp_asan::options::getOptions();
-         GuardedAlloc.init(opts);
-
-         ///std::cerr << "GwpAsan is initialized, the options are { Enabled: " << opts.Enabled
-         ///          << ", MaxSimultaneousAllocations: " << opts.MaxSimultaneousAllocations
-         ///          << ", SampleRate: " << opts.SampleRate << " }\n";
-    }
-} init_gwp_asan;
-#endif
-
 /// Replace default new/delete with memory tracking versions.
 /// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new
 ///     https://en.cppreference.com/w/cpp/memory/new/operator_delete

src/Core/ServerSettings.h

@@ -146,6 +146,7 @@ namespace DB
     M(UInt64, global_profiler_real_time_period_ns, 0, "Period for real clock timer of global profiler (in nanoseconds). Set 0 value to turn off the real clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(UInt64, global_profiler_cpu_time_period_ns, 0, "Period for CPU clock timer of global profiler (in nanoseconds). Set 0 value to turn off the CPU clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(Bool, enable_azure_sdk_logging, false, "Enables logging from Azure sdk", 0) \
+    M(Double, gwp_asan_force_sample_probability, 0, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \
 
 /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp
 

src/Daemon/BaseDaemon.cpp

@@ -6,6 +6,7 @@
 #include <Common/MemoryTracker.h>
 #include <Daemon/BaseDaemon.h>
 #include <Daemon/SentryWriter.h>
+#include <Common/GWPAsan.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -156,6 +157,12 @@ static void signalHandler(int sig, siginfo_t * info, void * context)
     const ucontext_t * signal_context = reinterpret_cast<ucontext_t *>(context);
     const StackTrace stack_trace(*signal_context);
 
+#if USE_GWP_ASAN
+    if (const auto fault_address = reinterpret_cast<uintptr_t>(info->si_addr);
+        GWPAsan::isGWPAsanError(fault_address))
+        GWPAsan::printReport(fault_address);
+#endif
+
     writeBinary(sig, out);
     writePODBinary(*info, out);
     writePODBinary(signal_context, out);

utils/check-style/check-style

@@ -323,6 +323,7 @@ std_cerr_cout_excludes=(
     src/Bridge/IBridge.cpp
     src/Daemon/BaseDaemon.cpp
     src/Loggers/Loggers.cpp
+    src/Common/GWPAsan.cpp
 )
 sources_with_std_cerr_cout=( $(
     find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \