From 709b4f42c82da439b0b3b2216fd6f56959411dd3 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 27 May 2020 22:11:04 +0300 Subject: [PATCH 01/82] Prototype sending crash reports on segfaults --- .gitmodules | 3 + CMakeLists.txt | 1 + base/daemon/BaseDaemon.cpp | 8 +- base/daemon/CMakeLists.txt | 8 +- base/daemon/SentryWriter.cpp | 107 +++++++++++++ base/daemon/SentryWriter.h | 21 +++ cmake/find/sentry.cmake | 19 +++ contrib/CMakeLists.txt | 14 +- contrib/curl-cmake/CMakeLists.txt | 2 + contrib/sentry-native | 1 + programs/server/Server.cpp | 2 + src/Common/StackTrace.cpp | 146 ++++++++++++------ src/Common/StackTrace.h | 22 ++- src/Common/TraceCollector.cpp | 2 +- .../System/StorageSystemStackTrace.cpp | 2 +- utils/check-style/check-include | 1 + 16 files changed, 298 insertions(+), 61 deletions(-) create mode 100644 base/daemon/SentryWriter.cpp create mode 100644 base/daemon/SentryWriter.h create mode 100644 cmake/find/sentry.cmake create mode 160000 contrib/sentry-native diff --git a/.gitmodules b/.gitmodules index 7f5d1307a6e..daa5d12a62c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -160,3 +160,6 @@ [submodule "contrib/fmtlib"] path = contrib/fmtlib url = https://github.com/fmtlib/fmt.git +[submodule "contrib/sentry-native"] + path = contrib/sentry-native + url = git@github.com:getsentry/sentry-native.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 53dfd1df1cb..79db4c624ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -365,6 +365,7 @@ include (cmake/find/fastops.cmake) include (cmake/find/orc.cmake) include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) +include (cmake/find/sentry.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 10c7173d5b1..f269c3923e0 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -222,6 +223,7 @@ public: DB::readPODBinary(stack_trace, in); DB::readBinary(thread_num, in); DB::readBinary(query_id, in); + stack_trace.resetFrames(); /// This allows to receive more signals if failure happens inside onFault function. /// Example: segfault while symbolizing stack trace. @@ -247,6 +249,7 @@ private: UInt32 thread_num, const std::string & query_id) const { + SentryWriter::onFault(sig, info, context, stack_trace); LOG_FATAL(log, "########################################"); { @@ -272,7 +275,7 @@ private: std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) - bare_stacktrace << ' ' << stack_trace.getFrames()[i]; + bare_stacktrace << ' ' << stack_trace.getFramePointers()[i]; LOG_FATAL(log, bare_stacktrace.str()); } @@ -511,6 +514,8 @@ void debugIncreaseOOMScore() {} void BaseDaemon::initialize(Application & self) { closeFDs(); + SentryWriter::initialize(); + task_manager = std::make_unique(); ServerApplication::initialize(self); @@ -518,7 +523,6 @@ void BaseDaemon::initialize(Application & self) argsToConfig(argv(), config(), PRIO_APPLICATION - 100); bool is_daemon = config().getBool("application.runAsDaemon", false); - if (is_daemon) { /** When creating pid file and looking for config, will search for paths relative to the working path of the program when started. diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 5d9a37dc75e..46fa4a0fe34 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -1,7 +1,13 @@ add_library (daemon BaseDaemon.cpp GraphiteWriter.cpp -) + SentryWriter.cpp) target_include_directories (daemon PUBLIC ..) target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) + +if (USE_SENTRY) + target_link_libraries (daemon PRIVATE curl) + target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) +# target_include_directories (daemon SYSTEM BEFORE PRIVATE ${SENTRY_INCLUDE_DIR}) +endif () \ No newline at end of file diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp new file mode 100644 index 00000000000..8859adc1c2e --- /dev/null +++ b/base/daemon/SentryWriter.cpp @@ -0,0 +1,107 @@ +#include + +#include +#if !defined(ARCADIA_BUILD) +# include "Common/config_version.h" +#endif + +#include + +namespace { + void setExtras() { + sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); + sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); + sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); + sentry_set_extra("version_revision", sentry_value_new_int32(VERSION_REVISION)); + sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR)); + sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); + sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); + } +} + +void SentryWriter::initialize() { + sentry_options_t * options = sentry_options_new(); + sentry_options_set_release(options, VERSION_STRING); + sentry_options_set_debug(options, 1); + sentry_init(options); + sentry_options_set_dsn(options, "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277"); + if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { + sentry_options_set_environment(options, "prod"); + } else { + sentry_options_set_environment(options, "test"); + } +} + +void SentryWriter::shutdown() { + sentry_shutdown(); +} + +void SentryWriter::onFault( + int sig, + const siginfo_t & info, + const ucontext_t & context, + const StackTrace & stack_trace + ) +{ + const std::string & error_message = signalToErrorMessage(sig, info, context); + sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str()); + sentry_set_tag("signal", strsignal(sig)); + sentry_set_tag("server_name", getFQDNOrHostName().c_str()); + sentry_set_extra("signal_number", sentry_value_new_int32(sig)); + setExtras(); + + sentry_value_t frames = sentry_value_new_list(); + + size_t stack_size = stack_trace.getSize(); + if (stack_size > 0) + { + size_t offset = stack_trace.getOffset(); + if (stack_size == 1) + { + offset = 1; + } + char instruction_addr[100]; + for (size_t i = stack_size - 1; i >= offset; --i) + { + const StackTrace::Frame & current_frame = stack_trace.getFrames().value()[i]; + sentry_value_t frame = sentry_value_new_object(); + unsigned long long frame_ptr = reinterpret_cast(current_frame.virtual_addr); + snprintf(instruction_addr, sizeof(instruction_addr), "0x%llx", frame_ptr); + sentry_value_set_by_key(frame, "instruction_addr", sentry_value_new_string(instruction_addr)); + + if (current_frame.symbol.has_value()) + { + sentry_value_set_by_key(frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); + } + + if (current_frame.file.has_value()) + { + sentry_value_set_by_key(frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); + } + + if (current_frame.line.has_value()) + { + sentry_value_set_by_key(frame, "lineno", sentry_value_new_int32(current_frame.line.value())); + } + + sentry_value_append(frames, frame); + } + } + + sentry_value_t stacktrace = sentry_value_new_object(); + sentry_value_set_by_key(stacktrace, "frames", frames); + + sentry_value_t thread = sentry_value_new_object(); + sentry_value_set_by_key(thread, "stacktrace", stacktrace); + + sentry_value_t values = sentry_value_new_list(); + sentry_value_append(values, thread); + + sentry_value_t threads = sentry_value_new_object(); + sentry_value_set_by_key(threads, "values", values); + + sentry_value_set_by_key(event, "threads", threads); + + sentry_capture_event(event); + shutdown(); +} diff --git a/base/daemon/SentryWriter.h b/base/daemon/SentryWriter.h new file mode 100644 index 00000000000..6c85ef04dd3 --- /dev/null +++ b/base/daemon/SentryWriter.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +#include + +class SentryWriter +{ +public: + SentryWriter() = delete; + + static void initialize(); + static void shutdown(); + static void onFault( + int sig, + const siginfo_t & info, + const ucontext_t & context, + const StackTrace & stack_trace + ); +}; diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake new file mode 100644 index 00000000000..f94b53ffb00 --- /dev/null +++ b/cmake/find/sentry.cmake @@ -0,0 +1,19 @@ +set (SENTRY_LIBRARY "sentry") +set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include") +if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") + message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive") + return() +endif () + +option (USE_SENTRY "Use Sentry" ON) + +set (BUILD_SHARED_LIBS OFF) +set (SENTRY_PIC OFF) +set (SENTRY_BACKEND "none") +set (SENTRY_TRANSPORT "curl") +set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) +set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) + +message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") + +include_directories("${SENTRY_INCLUDE_DIR}") \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 99f7be2cbb7..1d1d7756de2 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -274,7 +274,7 @@ if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () -if (USE_INTERNAL_AWS_S3_LIBRARY) +if (USE_INTERNAL_AWS_S3_LIBRARY OR USE_SENTRY) set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) @@ -286,12 +286,18 @@ if (USE_INTERNAL_AWS_S3_LIBRARY) set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES}) set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS}) set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH}) + + # The library is large - avoid bloat. + target_compile_options (curl PRIVATE -g0) +endif () + +if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) # The library is large - avoid bloat. target_compile_options (aws_s3 PRIVATE -g0) target_compile_options (aws_s3_checksums PRIVATE -g0) - target_compile_options (curl PRIVATE -g0) + endif () if (USE_BASE64) @@ -318,4 +324,8 @@ if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() +if (USE_SENTRY) + add_subdirectory (sentry-native) +endif() + add_subdirectory (fmtlib-cmake) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index d9805612ffe..d0f6a7773b0 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -1,4 +1,6 @@ set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl) +set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) +set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) set (SRCS ${CURL_DIR}/lib/file.c diff --git a/contrib/sentry-native b/contrib/sentry-native new file mode 160000 index 00000000000..3bfce2d17c1 --- /dev/null +++ b/contrib/sentry-native @@ -0,0 +1 @@ +Subproject commit 3bfce2d17c1b80fbbaae83bb5ef41c1b290d34fb diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c1a520030f4..8383fa2d9bf 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -60,6 +60,8 @@ #include #include #include "MySQLHandlerFactory.h" +#include + #if !defined(ARCADIA_BUILD) # include "config_core.h" diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 6d0b6a0f7d2..5cc8c43a27a 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -190,6 +190,63 @@ static void * getCallerAddress(const ucontext_t & context) #endif } +static void symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) +{ +#if defined(__ELF__) && !defined(__FreeBSD__) + + const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); + std::unordered_map dwarfs; + + for (size_t i = 0; i < offset; ++i) { + frames.value()[i].virtual_addr = frame_pointers[i]; + } + + for (size_t i = offset; i < size; ++i) + { + StackTrace::Frame & current_frame = frames.value()[i]; + current_frame.virtual_addr = frame_pointers[i]; + const auto * object = symbol_index.findObject(current_frame.virtual_addr); + uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; + current_frame.physical_addr = reinterpret_cast(uintptr_t(current_frame.virtual_addr) - virtual_offset); + + if (object) + { + current_frame.object = object->name; + if (std::filesystem::exists(current_frame.object.value())) + { + auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; + + DB::Dwarf::LocationInfo location; + if (dwarf_it->second.findAddress(uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) { + current_frame.file = location.file.toString(); + current_frame.line = location.line; + } + } + } + else + { + current_frame.object = "?"; + } + + const auto * symbol = symbol_index.findSymbol(current_frame.virtual_addr); + if (symbol) + { + int status = 0; + current_frame.symbol = demangle(symbol->name, status); + } + else + { + current_frame.symbol = "?"; + } + } +# else + for (size_t i = 0; i < size; ++i) { + frames.value()[i].virtual_addr = frame_pointers[i]; + } + UNUSED(offset); +#endif +} + StackTrace::StackTrace() { tryCapture(); @@ -203,7 +260,7 @@ StackTrace::StackTrace(const ucontext_t & signal_context) if (size == 0 && caller_address) { - frames[0] = caller_address; + frame_pointers[0] = caller_address; size = 1; } else @@ -212,7 +269,7 @@ StackTrace::StackTrace(const ucontext_t & signal_context) for (size_t i = 0; i < size; ++i) { - if (frames[i] == caller_address) + if (frame_pointers[i] == caller_address) { offset = i; break; @@ -229,8 +286,8 @@ void StackTrace::tryCapture() { size = 0; #if USE_UNWIND - size = unw_backtrace(frames.data(), capacity); - __msan_unpoison(frames.data(), size * sizeof(frames[0])); + size = unw_backtrace(frame_pointers.data(), capacity); + __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); #endif } @@ -244,102 +301,89 @@ size_t StackTrace::getOffset() const return offset; } -const StackTrace::Frames & StackTrace::getFrames() const +const StackTrace::FramePointers & StackTrace::getFramePointers() const { - return frames; + return frame_pointers; } +const StackTrace::Frames & StackTrace::getFrames() const +{ + if (!frames.has_value()) { + frames = {{}}; + symbolize(frame_pointers.data(), offset, size, frames); + } + return frames; +} static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t size, std::function callback) { if (size == 0) return callback(""); -#if defined(__ELF__) && !defined(__FreeBSD__) - const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); - std::unordered_map dwarfs; - std::stringstream out; for (size_t i = offset; i < size; ++i) { - const void * virtual_addr = frames[i]; - const auto * object = symbol_index.findObject(virtual_addr); - uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; - const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); - + const StackTrace::Frame& current_frame = frames.value()[i]; out << i << ". "; - if (object) + if (current_frame.file.has_value() && current_frame.line.has_value()) { - if (std::filesystem::exists(object->name)) - { - auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; - - DB::Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) - out << location.file.toString() << ":" << location.line << ": "; - } + out << current_frame.file.value() << ":" << current_frame.line.value() << ": "; } - const auto * symbol = symbol_index.findSymbol(virtual_addr); - if (symbol) + if (current_frame.symbol.has_value()) { - int status = 0; - out << demangle(symbol->name, status); + out << current_frame.symbol.value(); } - else - out << "?"; - out << " @ " << physical_addr; - out << " in " << (object ? object->name : "?"); + out << " @ " << current_frame.physical_addr; + if (current_frame.object.has_value()) { + out << " in " << current_frame.object.value(); + } callback(out.str()); out.str({}); } -#else - std::stringstream out; - - for (size_t i = offset; i < size; ++i) - { - const void * addr = frames[i]; - out << i << ". " << addr; - - callback(out.str()); - out.str({}); - } -#endif } -static std::string toStringImpl(const StackTrace::Frames & frames, size_t offset, size_t size) +static std::string toStringImpl(const void * const * frame_pointers, size_t offset, size_t size) { std::stringstream out; + StackTrace::Frames frames{}; + frames = {{}}; + symbolize(frame_pointers, offset, size, frames); toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); } void StackTrace::toStringEveryLine(std::function callback) const { - toStringEveryLineImpl(frames, offset, size, std::move(callback)); + toStringEveryLineImpl(getFrames(), offset, size, std::move(callback)); } +void StackTrace::resetFrames() { + frames.reset(); +} + + std::string StackTrace::toString() const { /// Calculation of stack trace text is extremely slow. /// We use simple cache because otherwise the server could be overloaded by trash queries. static SimpleCache func_cached; - return func_cached(frames, offset, size); + return func_cached(frame_pointers.data(), offset, size); } -std::string StackTrace::toString(void ** frames_, size_t offset, size_t size) +std::string StackTrace::toString(void ** frame_pointers, size_t offset, size_t size) { __msan_unpoison(frames_, size * sizeof(*frames_)); - StackTrace::Frames frames_copy{}; + StackTrace::FramePointers frame_pointers_copy{}; for (size_t i = 0; i < size; ++i) - frames_copy[i] = frames_[i]; + frame_pointers_copy[i] = frame_pointers[i]; static SimpleCache func_cached; - return func_cached(frames_copy, offset, size); + return func_cached(frame_pointers_copy.data(), offset, size); } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 401c8344f2d..27b2c44dd94 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -23,8 +25,18 @@ struct NoCapture class StackTrace { public: + struct Frame + { + const void * virtual_addr = nullptr; + void * physical_addr = nullptr; + std::optional symbol; + std::optional object; + std::optional file; + std::optional line; + }; static constexpr size_t capacity = 32; - using Frames = std::array; + using FramePointers = std::array; + using Frames = std::optional>; /// Tries to capture stack trace StackTrace(); @@ -38,19 +50,23 @@ public: size_t getSize() const; size_t getOffset() const; + const FramePointers & getFramePointers() const; const Frames & getFrames() const; std::string toString() const; - static std::string toString(void ** frames, size_t offset, size_t size); + static std::string toString(void ** frame_pointers, size_t offset, size_t size); void toStringEveryLine(std::function callback) const; + void resetFrames(); + protected: void tryCapture(); size_t size = 0; size_t offset = 0; /// How many frames to skip while displaying. - Frames frames{}; + FramePointers frame_pointers{}; + mutable Frames frames{}; }; std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context); diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index 7df06dc7892..f5bdfd2b826 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -81,7 +81,7 @@ void TraceCollector::collect(TraceType trace_type, const StackTrace & stack_trac size_t stack_trace_offset = stack_trace.getOffset(); writeIntBinary(UInt8(stack_trace_size - stack_trace_offset), out); for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) - writePODBinary(stack_trace.getFrames()[i], out); + writePODBinary(stack_trace.getFramePointers()[i], out); writePODBinary(trace_type, out); writePODBinary(thread_id, out); diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index a8966ad0307..bdce70894d5 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -198,7 +198,7 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, const Conte Array arr; arr.reserve(stack_trace_size - stack_trace_offset); for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) - arr.emplace_back(reinterpret_cast(stack_trace->getFrames()[i])); + arr.emplace_back(reinterpret_cast(stack_trace->getFramePointers()[i])); res_columns[0]->insert(tid); res_columns[1]->insertData(query_id_data, query_id_size); diff --git a/utils/check-style/check-include b/utils/check-style/check-include index 211172979bd..35f94d6e706 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -59,6 +59,7 @@ inc="-I. \ -I./contrib/lz4/lib \ -I./contrib/hyperscan/src \ -I./contrib/simdjson/include \ +-I./contrib/sentry-native/include \ -I./src \ -I${BUILD_DIR}/src" From 31123236cb359f1783dcadf8c3062ddb1ca6b8cf Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 27 May 2020 23:15:33 +0300 Subject: [PATCH 02/82] Settings for crash report opt-in --- base/daemon/BaseDaemon.cpp | 2 +- base/daemon/SentryWriter.cpp | 164 +++++++++++++++++++++-------------- base/daemon/SentryWriter.h | 4 +- src/Common/config.h.in | 1 + 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index f269c3923e0..a8a79827552 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -514,7 +514,6 @@ void debugIncreaseOOMScore() {} void BaseDaemon::initialize(Application & self) { closeFDs(); - SentryWriter::initialize(); task_manager = std::make_unique(); ServerApplication::initialize(self); @@ -533,6 +532,7 @@ void BaseDaemon::initialize(Application & self) } reloadConfiguration(); + SentryWriter::initialize(config()); /// This must be done before creation of any files (including logs). mode_t umask_num = 0027; diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 8859adc1c2e..5c7d6eadd98 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -1,14 +1,21 @@ #include +#include #include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" #endif +#if USE_SENTRY #include +#endif + namespace { + static bool initialized = false; + void setExtras() { +#if USE_SENTRY sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); @@ -16,24 +23,47 @@ namespace { sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR)); sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); +#endif } } -void SentryWriter::initialize() { - sentry_options_t * options = sentry_options_new(); - sentry_options_set_release(options, VERSION_STRING); - sentry_options_set_debug(options, 1); - sentry_init(options); - sentry_options_set_dsn(options, "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277"); - if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { - sentry_options_set_environment(options, "prod"); - } else { - sentry_options_set_environment(options, "test"); +void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { +#if USE_SENTRY + bool enabled = false; + if (config.getBool("send_crash_reports.enabled", false)) + { + if ((strlen(VERSION_OFFICIAL) > 0) || config.getBool("send_crash_reports.debug", false)) + { + enabled = true; + } } + if (enabled) + { + const std::string & endpoint = config.getString( + "send_crash_reports.endpoint", + "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277" + ); + sentry_options_t * options = sentry_options_new(); + sentry_options_set_release(options, VERSION_STRING); + sentry_options_set_debug(options, 1); + sentry_init(options); + sentry_options_set_dsn(options, endpoint.c_str()); + if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { + sentry_options_set_environment(options, "prod"); + } else { + sentry_options_set_environment(options, "test"); + } + initialized = true; + } +#endif } void SentryWriter::shutdown() { - sentry_shutdown(); +#if USE_SENTRY + if (initialized) { + sentry_shutdown(); + } +#endif } void SentryWriter::onFault( @@ -43,65 +73,71 @@ void SentryWriter::onFault( const StackTrace & stack_trace ) { - const std::string & error_message = signalToErrorMessage(sig, info, context); - sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str()); - sentry_set_tag("signal", strsignal(sig)); - sentry_set_tag("server_name", getFQDNOrHostName().c_str()); - sentry_set_extra("signal_number", sentry_value_new_int32(sig)); - setExtras(); - - sentry_value_t frames = sentry_value_new_list(); - - size_t stack_size = stack_trace.getSize(); - if (stack_size > 0) +#if USE_SENTRY + if (initialized) { - size_t offset = stack_trace.getOffset(); - if (stack_size == 1) + const std::string & error_message = signalToErrorMessage(sig, info, context); + sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str()); + sentry_set_tag("signal", strsignal(sig)); + sentry_set_tag("server_name", getFQDNOrHostName().c_str()); + sentry_set_extra("signal_number", sentry_value_new_int32(sig)); + setExtras(); + + /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/stacktrace/ + sentry_value_t frames = sentry_value_new_list(); + size_t stack_size = stack_trace.getSize(); + if (stack_size > 0) { - offset = 1; + size_t offset = stack_trace.getOffset(); + if (stack_size == 1) + { + offset = 1; + } + char instruction_addr[100]; + for (size_t i = stack_size - 1; i >= offset; --i) + { + const StackTrace::Frame & current_frame = stack_trace.getFrames().value()[i]; + sentry_value_t frame = sentry_value_new_object(); + unsigned long long frame_ptr = reinterpret_cast(current_frame.virtual_addr); + snprintf(instruction_addr, sizeof(instruction_addr), "0x%llx", frame_ptr); + sentry_value_set_by_key(frame, "instruction_addr", sentry_value_new_string(instruction_addr)); + + if (current_frame.symbol.has_value()) + { + sentry_value_set_by_key(frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); + } + + if (current_frame.file.has_value()) + { + sentry_value_set_by_key(frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); + } + + if (current_frame.line.has_value()) + { + sentry_value_set_by_key(frame, "lineno", sentry_value_new_int32(current_frame.line.value())); + } + + sentry_value_append(frames, frame); + } } - char instruction_addr[100]; - for (size_t i = stack_size - 1; i >= offset; --i) - { - const StackTrace::Frame & current_frame = stack_trace.getFrames().value()[i]; - sentry_value_t frame = sentry_value_new_object(); - unsigned long long frame_ptr = reinterpret_cast(current_frame.virtual_addr); - snprintf(instruction_addr, sizeof(instruction_addr), "0x%llx", frame_ptr); - sentry_value_set_by_key(frame, "instruction_addr", sentry_value_new_string(instruction_addr)); - if (current_frame.symbol.has_value()) - { - sentry_value_set_by_key(frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); - } + /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/threads/ + sentry_value_t stacktrace = sentry_value_new_object(); + sentry_value_set_by_key(stacktrace, "frames", frames); - if (current_frame.file.has_value()) - { - sentry_value_set_by_key(frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); - } + sentry_value_t thread = sentry_value_new_object(); + sentry_value_set_by_key(thread, "stacktrace", stacktrace); - if (current_frame.line.has_value()) - { - sentry_value_set_by_key(frame, "lineno", sentry_value_new_int32(current_frame.line.value())); - } + sentry_value_t values = sentry_value_new_list(); + sentry_value_append(values, thread); - sentry_value_append(frames, frame); - } + sentry_value_t threads = sentry_value_new_object(); + sentry_value_set_by_key(threads, "values", values); + + sentry_value_set_by_key(event, "threads", threads); + + sentry_capture_event(event); + shutdown(); } - - sentry_value_t stacktrace = sentry_value_new_object(); - sentry_value_set_by_key(stacktrace, "frames", frames); - - sentry_value_t thread = sentry_value_new_object(); - sentry_value_set_by_key(thread, "stacktrace", stacktrace); - - sentry_value_t values = sentry_value_new_list(); - sentry_value_append(values, thread); - - sentry_value_t threads = sentry_value_new_object(); - sentry_value_set_by_key(threads, "values", values); - - sentry_value_set_by_key(event, "threads", threads); - - sentry_capture_event(event); - shutdown(); +#endif } diff --git a/base/daemon/SentryWriter.h b/base/daemon/SentryWriter.h index 6c85ef04dd3..ee45ae4f203 100644 --- a/base/daemon/SentryWriter.h +++ b/base/daemon/SentryWriter.h @@ -3,6 +3,8 @@ #include #include +#include + #include class SentryWriter @@ -10,7 +12,7 @@ class SentryWriter public: SentryWriter() = delete; - static void initialize(); + static void initialize(Poco::Util::LayeredConfiguration & config); static void shutdown(); static void onFault( int sig, diff --git a/src/Common/config.h.in b/src/Common/config.h.in index df2359c1c29..dd6263c3948 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -9,4 +9,5 @@ #cmakedefine01 USE_BROTLI #cmakedefine01 USE_UNWIND #cmakedefine01 USE_OPENCL +#cmakedefine01 USE_SENTRY #cmakedefine01 CLICKHOUSE_SPLIT_BINARY From 52e4a0293d622072bbd8d9f09d37bc7257b83174 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 27 May 2020 23:21:53 +0300 Subject: [PATCH 03/82] Keep sentry-native in debug mode only under setting --- base/daemon/SentryWriter.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 5c7d6eadd98..7e2a95c8369 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -30,9 +30,10 @@ namespace { void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { #if USE_SENTRY bool enabled = false; + bool debug = config.getBool("send_crash_reports.debug", false); if (config.getBool("send_crash_reports.enabled", false)) { - if ((strlen(VERSION_OFFICIAL) > 0) || config.getBool("send_crash_reports.debug", false)) + if (debug || (strlen(VERSION_OFFICIAL) > 0)) { enabled = true; } @@ -45,7 +46,10 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { ); sentry_options_t * options = sentry_options_new(); sentry_options_set_release(options, VERSION_STRING); - sentry_options_set_debug(options, 1); + if (debug) + { + sentry_options_set_debug(options, 1); + } sentry_init(options); sentry_options_set_dsn(options, endpoint.c_str()); if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { From d9bb3ef91ba801f4752dde77032b43e892395e14 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 22:48:32 +0300 Subject: [PATCH 04/82] Add logging and adjust initialization --- base/daemon/BaseDaemon.cpp | 2 +- base/daemon/SentryWriter.cpp | 38 ++++++++++++++++++++++++++++++++++-- base/daemon/ya.make | 1 + 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index a8a79827552..4fd5bfa1379 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -532,7 +532,6 @@ void BaseDaemon::initialize(Application & self) } reloadConfiguration(); - SentryWriter::initialize(config()); /// This must be done before creation of any files (including logs). mode_t umask_num = 0027; @@ -658,6 +657,7 @@ void BaseDaemon::initialize(Application & self) void BaseDaemon::initializeTerminationAndSignalProcessing() { + SentryWriter::initialize(config()); std::set_terminate(terminate_handler); /// We want to avoid SIGPIPE when working with sockets and pipes, and just handle return value/errno instead. diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 7e2a95c8369..d5c2766cf21 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -1,7 +1,11 @@ #include +#include +#include + #include #include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" #endif @@ -44,20 +48,45 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { "send_crash_reports.endpoint", "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277" ); + const std::string & temp_folder_path = config.getString( + "send_crash_reports.tmp_path", + config.getString("tmp_path", Poco::Path::temp()) + "sentry/" + ); + Poco::File(temp_folder_path).createDirectories(); + sentry_options_t * options = sentry_options_new(); sentry_options_set_release(options, VERSION_STRING); if (debug) { sentry_options_set_debug(options, 1); } - sentry_init(options); sentry_options_set_dsn(options, endpoint.c_str()); + sentry_options_set_database_path(options, temp_folder_path.c_str()); if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { sentry_options_set_environment(options, "prod"); } else { sentry_options_set_environment(options, "test"); } - initialized = true; + int init_status = sentry_init(options); + if (!init_status) + { + initialized = true; + LOG_INFO( + &Logger::get("SentryWriter"), + "Sending crash reports is initialized with {} endpoint and {} temp folder", + endpoint, + temp_folder_path + ); + } + else + { + LOG_WARNING(&Logger::get("SentryWriter"), "Sending crash reports failed to initialized with {} status", init_status); + } + + } + else + { + LOG_INFO(&Logger::get("SentryWriter"), "Sending crash reports is disabled"); } #endif } @@ -140,8 +169,13 @@ void SentryWriter::onFault( sentry_value_set_by_key(event, "threads", threads); + LOG_INFO(&Logger::get("SentryWriter"), "Sending crash report"); sentry_capture_event(event); shutdown(); } + else + { + LOG_INFO(&Logger::get("SentryWriter"), "Not sending crash report"); + } #endif } diff --git a/base/daemon/ya.make b/base/daemon/ya.make index 1c72af3ed53..125417adca5 100644 --- a/base/daemon/ya.make +++ b/base/daemon/ya.make @@ -9,6 +9,7 @@ PEERDIR( SRCS( BaseDaemon.cpp GraphiteWriter.cpp + SentryWriter.cpp ) END() From 4ef322274d117ecb6d04f79c4f73d0447b961c64 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 22:53:16 +0300 Subject: [PATCH 05/82] Add integration test --- .gitignore | 1 + programs/server/config.xml | 6 +++ .../configs/config_send_crash_reports.xml | 8 ++++ .../test_send_crash_reports/http_server.py | 43 ++++++++++++++++++ .../test_send_crash_reports/test.py | 44 +++++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 tests/integration/test_send_crash_reports/configs/config_send_crash_reports.xml create mode 100644 tests/integration/test_send_crash_reports/http_server.py create mode 100644 tests/integration/test_send_crash_reports/test.py diff --git a/.gitignore b/.gitignore index 6bd57911ac8..afb4e67a1b8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /build /build_* /build-* +/tests/venv /docs/build /docs/publish diff --git a/programs/server/config.xml b/programs/server/config.xml index e16af9d75d7..d07f20aa0e0 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -42,6 +42,12 @@ --> + + + + false + + 8123 9000 diff --git a/tests/integration/test_send_crash_reports/configs/config_send_crash_reports.xml b/tests/integration/test_send_crash_reports/configs/config_send_crash_reports.xml new file mode 100644 index 00000000000..10f559b0054 --- /dev/null +++ b/tests/integration/test_send_crash_reports/configs/config_send_crash_reports.xml @@ -0,0 +1,8 @@ + + + + true + true + http://6f33034cfe684dd7a3ab9875e57b1c8d@localhost:9500/5226277 + + diff --git a/tests/integration/test_send_crash_reports/http_server.py b/tests/integration/test_send_crash_reports/http_server.py new file mode 100644 index 00000000000..e3fa2e1cb57 --- /dev/null +++ b/tests/integration/test_send_crash_reports/http_server.py @@ -0,0 +1,43 @@ +import BaseHTTPServer + +RESULT_PATH = '/result.txt' + +class SentryHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + post_data = self.__read_and_decode_post_data() + with open(RESULT_PATH, 'w') as f: + if self.headers.get("content-type") != "application/x-sentry-envelope": + f.write("INCORRECT_CONTENT_TYPE") + elif self.headers.get("content-length") < 3000: + f.write("INCORRECT_CONTENT_LENGTH") + elif '"http://6f33034cfe684dd7a3ab9875e57b1c8d@localhost:9500/5226277"' not in post_data: + f.write('INCORRECT_POST_DATA') + else: + f.write("OK") + self.send_response(200) + + def __read_and_decode_post_data(self): + transfer_encoding = self.headers.get("transfer-Encoding") + decoded = "" + if transfer_encoding == "chunked": + while True: + s = self.rfile.readline() + chunk_length = int(s, 16) + if not chunk_length: + break + decoded += self.rfile.read(chunk_length) + self.rfile.readline() + else: + content_length = int(self.headers.get("content-length", 0)) + decoded = self.rfile.read(content_length) + return decoded + + +if __name__ == "__main__": + with open(RESULT_PATH, 'w') as f: + f.write("INITIAL_STATE") + httpd = BaseHTTPServer.HTTPServer(("localhost", 9500,), SentryHandler) + try: + httpd.serve_forever() + finally: + httpd.server_close() \ No newline at end of file diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py new file mode 100644 index 00000000000..f9e95f953d0 --- /dev/null +++ b/tests/integration/test_send_crash_reports/test.py @@ -0,0 +1,44 @@ +import os +import time + +import pytest + +import helpers.cluster +import helpers.test_tools +import http_server + + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +@pytest.fixture(scope="module") +def started_node(): + cluster = helpers.cluster.ClickHouseCluster(__file__) + try: + node = cluster.add_instance("node", main_configs=[ + os.path.join(SCRIPT_DIR, "configs", "config_send_crash_reports.xml") + ]) + cluster.start() + yield node + finally: + cluster.shutdown() + + +def test_send_segfault(started_node,): + started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "http_server.py"), "/http_server.py") + started_node.exec_in_container(["bash", "-c", "python2 /http_server.py"], detach=True, user="root") + time.sleep(0.5) + started_node.exec_in_container(["bash", "-c", "pkill -11 clickhouse"], user="root") + + result = None + for attempt in range(1, 6): + time.sleep(0.25 * attempt) + result = started_node.exec_in_container(['cat', http_server.RESULT_PATH], user='root') + if result == 'OK': + break + elif result == 'INITIAL_STATE': + continue + elif result: + assert False, 'Unexpected state: ' + result + + assert result == 'OK', 'Crash report not sent' From 0386e526b2c7cbf13e017f5445ea79eb4f24f67a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 23:03:59 +0300 Subject: [PATCH 06/82] grammar --- programs/server/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index d07f20aa0e0..6086fcd7b1d 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -43,7 +43,7 @@ - + false From a84123195b7fe4677c417de9fe5483c5c283ec13 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 23:08:05 +0300 Subject: [PATCH 07/82] adjust comments --- base/daemon/CMakeLists.txt | 1 - base/daemon/SentryWriter.h | 1 + src/Common/config.h.in | 3 --- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 46fa4a0fe34..0b6a7188c83 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -9,5 +9,4 @@ target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickh if (USE_SENTRY) target_link_libraries (daemon PRIVATE curl) target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) -# target_include_directories (daemon SYSTEM BEFORE PRIVATE ${SENTRY_INCLUDE_DIR}) endif () \ No newline at end of file diff --git a/base/daemon/SentryWriter.h b/base/daemon/SentryWriter.h index ee45ae4f203..0b3f1ddd2b7 100644 --- a/base/daemon/SentryWriter.h +++ b/base/daemon/SentryWriter.h @@ -7,6 +7,7 @@ #include +/// Sends crash reports to ClickHouse core developer team via https://sentry.io class SentryWriter { public: diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 08fa03d659f..ed818b53167 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -9,9 +9,6 @@ #cmakedefine01 USE_BROTLI #cmakedefine01 USE_UNWIND #cmakedefine01 USE_OPENCL -<<<<<<< HEAD #cmakedefine01 USE_SENTRY -======= #cmakedefine01 USE_GRPC ->>>>>>> a4e40fb5f209539cfee6af5da7f27c1c96e02eac #cmakedefine01 CLICKHOUSE_SPLIT_BINARY From d0339413993371c37577ce513ecf0555683878b8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 23:20:28 +0300 Subject: [PATCH 08/82] try to fix merge issues --- contrib/grpc | 2 +- contrib/jemalloc | 2 +- programs/server/Server.cpp | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/contrib/grpc b/contrib/grpc index c1d176528fd..8aea4e168e7 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit c1d176528fd8da9dd4066d16554bcd216d29033f +Subproject commit 8aea4e168e78f3eb9828080740fc8cb73d53bf79 diff --git a/contrib/jemalloc b/contrib/jemalloc index cd2931ad9bb..ea6b3e973b4 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff +Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756 diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 77dc5305fa8..ce1d35e65d4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -59,13 +59,7 @@ #include #include #include -<<<<<<< HEAD -#include "MySQLHandlerFactory.h" -#include - -======= #include ->>>>>>> a4e40fb5f209539cfee6af5da7f27c1c96e02eac #if !defined(ARCADIA_BUILD) # include "config_core.h" From b6e4a2ec61c928a433037fefa0657df7ebf8b8ac Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 29 May 2020 23:21:53 +0300 Subject: [PATCH 09/82] one more merge issue --- contrib/cppkafka | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cppkafka b/contrib/cppkafka index 9b184d881c1..f555ee36aaa 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 9b184d881c15cc50784b28688c7c99d3d764db24 +Subproject commit f555ee36aaa74d17ca0dab3ce472070a610b2966 From 69dedcbe21bd323f3d87508ba78f36b587a7dff5 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 00:28:55 +0300 Subject: [PATCH 10/82] Move sending crash reports below logging --- base/daemon/BaseDaemon.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 4fd5bfa1379..72da1984287 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -249,7 +249,6 @@ private: UInt32 thread_num, const std::string & query_id) const { - SentryWriter::onFault(sig, info, context, stack_trace); LOG_FATAL(log, "########################################"); { @@ -282,6 +281,9 @@ private: /// Write symbolized stack trace line by line for better grep-ability. stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); + + /// Send crash report if configured + SentryWriter::onFault(sig, info, context, stack_trace); } }; From f88b85625a44cbcb1628dc76283567c6fceeedd7 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 00:36:47 +0300 Subject: [PATCH 11/82] style --- base/daemon/SentryWriter.cpp | 71 +++++++++++++++++------------------- src/Common/StackTrace.cpp | 52 +++++++++++++------------- 2 files changed, 61 insertions(+), 62 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index d5c2766cf21..7bbf3c62e97 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -3,35 +3,38 @@ #include #include -#include #include #include #if !defined(ARCADIA_BUILD) -# include "Common/config_version.h" +# include "Common/config_version.h" +# include #endif #if USE_SENTRY -#include +# include #endif -namespace { - static bool initialized = false; +namespace +{ +static bool initialized = false; - void setExtras() { +void setExtras() +{ #if USE_SENTRY - sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); - sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); - sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); - sentry_set_extra("version_revision", sentry_value_new_int32(VERSION_REVISION)); - sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR)); - sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); - sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); + sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); + sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); + sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); + sentry_set_extra("version_revision", sentry_value_new_int32(VERSION_REVISION)); + sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR)); + sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); + sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); #endif - } +} } -void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { +void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) +{ #if USE_SENTRY bool enabled = false; bool debug = config.getBool("send_crash_reports.debug", false); @@ -44,14 +47,10 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { } if (enabled) { - const std::string & endpoint = config.getString( - "send_crash_reports.endpoint", - "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277" - ); - const std::string & temp_folder_path = config.getString( - "send_crash_reports.tmp_path", - config.getString("tmp_path", Poco::Path::temp()) + "sentry/" - ); + const std::string & endpoint + = config.getString("send_crash_reports.endpoint", "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277"); + const std::string & temp_folder_path + = config.getString("send_crash_reports.tmp_path", config.getString("tmp_path", Poco::Path::temp()) + "sentry/"); Poco::File(temp_folder_path).createDirectories(); sentry_options_t * options = sentry_options_new(); @@ -62,9 +61,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { } sentry_options_set_dsn(options, endpoint.c_str()); sentry_options_set_database_path(options, temp_folder_path.c_str()); - if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) { + if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts")) + { sentry_options_set_environment(options, "prod"); - } else { + } + else + { sentry_options_set_environment(options, "test"); } int init_status = sentry_init(options); @@ -75,14 +77,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { &Logger::get("SentryWriter"), "Sending crash reports is initialized with {} endpoint and {} temp folder", endpoint, - temp_folder_path - ); + temp_folder_path); } else { LOG_WARNING(&Logger::get("SentryWriter"), "Sending crash reports failed to initialized with {} status", init_status); } - } else { @@ -91,20 +91,17 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { #endif } -void SentryWriter::shutdown() { +void SentryWriter::shutdown() +{ #if USE_SENTRY - if (initialized) { + if (initialized) + { sentry_shutdown(); } #endif } -void SentryWriter::onFault( - int sig, - const siginfo_t & info, - const ucontext_t & context, - const StackTrace & stack_trace - ) +void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace) { #if USE_SENTRY if (initialized) @@ -178,4 +175,4 @@ void SentryWriter::onFault( LOG_INFO(&Logger::get("SentryWriter"), "Not sending crash report"); } #endif -} +} \ No newline at end of file diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 5cc8c43a27a..2fd554fd008 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -1,12 +1,12 @@ #include +#include #include #include -#include #include +#include #include #include -#include #include #include @@ -26,8 +26,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext std::stringstream error; switch (sig) { - case SIGSEGV: - { + case SIGSEGV: { /// Print info about address and reason. if (nullptr == info.si_addr) error << "Address: NULL pointer."; @@ -59,8 +58,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGBUS: - { + case SIGBUS: { switch (info.si_code) { case BUS_ADRALN: @@ -92,8 +90,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGILL: - { + case SIGILL: { switch (info.si_code) { case ILL_ILLOPC: @@ -127,8 +124,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGFPE: - { + case SIGFPE: { switch (info.si_code) { case FPE_INTDIV: @@ -162,8 +158,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGTSTP: - { + case SIGTSTP: { error << "This is a signal used for debugging purposes by the user."; break; } @@ -176,13 +171,13 @@ static void * getCallerAddress(const ucontext_t & context) { #if defined(__x86_64__) /// Get the address at the time the signal was raised from the RIP (x86-64) -#if defined(__FreeBSD__) +# if defined(__FreeBSD__) return reinterpret_cast(context.uc_mcontext.mc_rip); -#elif defined(__APPLE__) +# elif defined(__APPLE__) return reinterpret_cast(context.uc_mcontext->__ss.__rip); -#else +# else return reinterpret_cast(context.uc_mcontext.gregs[REG_RIP]); -#endif +# endif #elif defined(__aarch64__) return reinterpret_cast(context.uc_mcontext.pc); #else @@ -197,7 +192,8 @@ static void symbolize(const void * const * frame_pointers, size_t offset, size_t const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); std::unordered_map dwarfs; - for (size_t i = 0; i < offset; ++i) { + for (size_t i = 0; i < offset; ++i) + { frames.value()[i].virtual_addr = frame_pointers[i]; } @@ -217,7 +213,8 @@ static void symbolize(const void * const * frame_pointers, size_t offset, size_t auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; DB::Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) { + if (dwarf_it->second.findAddress(uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) + { current_frame.file = location.file.toString(); current_frame.line = location.line; } @@ -239,8 +236,9 @@ static void symbolize(const void * const * frame_pointers, size_t offset, size_t current_frame.symbol = "?"; } } -# else - for (size_t i = 0; i < size; ++i) { +#else + for (size_t i = 0; i < size; ++i) + { frames.value()[i].virtual_addr = frame_pointers[i]; } UNUSED(offset); @@ -308,14 +306,16 @@ const StackTrace::FramePointers & StackTrace::getFramePointers() const const StackTrace::Frames & StackTrace::getFrames() const { - if (!frames.has_value()) { + if (!frames.has_value()) + { frames = {{}}; symbolize(frame_pointers.data(), offset, size, frames); } return frames; } -static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t size, std::function callback) +static void +toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t size, std::function callback) { if (size == 0) return callback(""); @@ -324,7 +324,7 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs for (size_t i = offset; i < size; ++i) { - const StackTrace::Frame& current_frame = frames.value()[i]; + const StackTrace::Frame & current_frame = frames.value()[i]; out << i << ". "; if (current_frame.file.has_value() && current_frame.line.has_value()) @@ -338,7 +338,8 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs } out << " @ " << current_frame.physical_addr; - if (current_frame.object.has_value()) { + if (current_frame.object.has_value()) + { out << " in " << current_frame.object.value(); } @@ -362,7 +363,8 @@ void StackTrace::toStringEveryLine(std::function call toStringEveryLineImpl(getFrames(), offset, size, std::move(callback)); } -void StackTrace::resetFrames() { +void StackTrace::resetFrames() +{ frames.reset(); } From 444026494f9383d465ab9b9611c8bfc935661d85 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 00:52:49 +0300 Subject: [PATCH 12/82] brief docs --- .../settings.md | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 5961c701283..a103473a4ea 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -307,11 +307,11 @@ Logging settings. Keys: -- level – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. -- log – The log file. Contains all the entries according to `level`. -- errorlog – Error log file. -- size – Size of the file. Applies to `log`and`errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place. -- count – The number of archived log files that ClickHouse stores. +- `level` – Logging level. Acceptable values: `trace`, `debug`, `information`, `warning`, `error`. +- `log` – The log file. Contains all the entries according to `level`. +- `errorlog` – Error log file. +- `size` – Size of the file. Applies to `log`and`errorlog`. Once the file reaches `size`, ClickHouse archives and renames it, and creates a new log file in its place. +- `count` – The number of archived log files that ClickHouse stores. **Example** @@ -348,6 +348,27 @@ Keys: Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON otherwise.` - format – Message format. Possible values: `bsd` and `syslog.` +## send_crash_reports {#server_configuration_parameters-logger} + +Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). +Enabling it, especially in pre-production environments, is strongly appreciated. + +Keys: + +- `enabled` – Boolean flag to enable the feature. Set to `true` to allow sending crash reports. +- `endpoint` – Overrides the Sentry endpoint. +- `debug` - Sets the Sentry client into debug mode. +- `tmp_path` - Filesystem path for temporary crash report state. + + +**Recommended way to use** + +``` xml + + true + +``` + ## macros {#macros} Parameter substitutions for replicated tables. From 95ca1c648da4e95ec1cb252afd6e79216f4f5aec Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 10:59:43 +0300 Subject: [PATCH 13/82] fix __msan_unpoison --- src/Common/StackTrace.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 2fd554fd008..e38bfa25dff 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -378,13 +378,13 @@ std::string StackTrace::toString() const return func_cached(frame_pointers.data(), offset, size); } -std::string StackTrace::toString(void ** frame_pointers, size_t offset, size_t size) +std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t size) { - __msan_unpoison(frames_, size * sizeof(*frames_)); + __msan_unpoison(frame_pointers_, size * sizeof(*frame_pointers_)); StackTrace::FramePointers frame_pointers_copy{}; for (size_t i = 0; i < size; ++i) - frame_pointers_copy[i] = frame_pointers[i]; + frame_pointers_copy[i] = frame_pointers_[i]; static SimpleCache func_cached; return func_cached(frame_pointers_copy.data(), offset, size); From 6dfe44f437c393ded72f8859ca16d9625b8fbb53 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 11:01:15 +0300 Subject: [PATCH 14/82] style --- src/Common/StackTrace.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index e38bfa25dff..aa78ab62f9b 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -26,7 +26,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext std::stringstream error; switch (sig) { - case SIGSEGV: { + case SIGSEGV: + { /// Print info about address and reason. if (nullptr == info.si_addr) error << "Address: NULL pointer."; @@ -58,7 +59,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGBUS: { + case SIGBUS: + { switch (info.si_code) { case BUS_ADRALN: @@ -90,7 +92,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGILL: { + case SIGILL: + { switch (info.si_code) { case ILL_ILLOPC: @@ -124,7 +127,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGFPE: { + case SIGFPE: + { switch (info.si_code) { case FPE_INTDIV: @@ -158,7 +162,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext break; } - case SIGTSTP: { + case SIGTSTP: + { error << "This is a signal used for debugging purposes by the user."; break; } From 77d8c9bacae6b833a28a85ab45442355c0f4b2df Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 11:02:13 +0300 Subject: [PATCH 15/82] Add anonymize option and version tag --- base/daemon/SentryWriter.cpp | 17 +++++++++++++---- cmake/version.cmake | 1 + .../server-configuration-parameters/settings.md | 1 + src/Common/config_version.h.in | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 7bbf3c62e97..878ce6548aa 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -18,10 +18,16 @@ namespace { static bool initialized = false; +static bool anonymize = false; void setExtras() { #if USE_SENTRY + if (!anonymize) + { + sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); + } + sentry_set_tag("version", VERSION_STRING_SHORT); sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); @@ -69,15 +75,19 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { sentry_options_set_environment(options, "test"); } + int init_status = sentry_init(options); if (!init_status) { initialized = true; + anonymize = config.getBool("send_crash_reports.anonymize", false); + const std::string& anonymize_status = anonymize ? " (anonymized)" : ""; LOG_INFO( &Logger::get("SentryWriter"), - "Sending crash reports is initialized with {} endpoint and {} temp folder", + "Sending crash reports is initialized with {} endpoint and {} temp folder{}", endpoint, - temp_folder_path); + temp_folder_path, + anonymize_status); } else { @@ -109,7 +119,6 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c const std::string & error_message = signalToErrorMessage(sig, info, context); sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str()); sentry_set_tag("signal", strsignal(sig)); - sentry_set_tag("server_name", getFQDNOrHostName().c_str()); sentry_set_extra("signal_number", sentry_value_new_int32(sig)); setExtras(); @@ -175,4 +184,4 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c LOG_INFO(&Logger::get("SentryWriter"), "Not sending crash report"); } #endif -} \ No newline at end of file +} diff --git a/cmake/version.cmake b/cmake/version.cmake index eea17f68c47..963f291c0f3 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -14,6 +14,7 @@ endif () set (VERSION_NAME "${PROJECT_NAME}") set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}") set (VERSION_SO "${VERSION_STRING}") +set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}") math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index a103473a4ea..ba8f3df9ad0 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -357,6 +357,7 @@ Keys: - `enabled` – Boolean flag to enable the feature. Set to `true` to allow sending crash reports. - `endpoint` – Overrides the Sentry endpoint. +- `anonymize` - Avoid attaching the server hostname to crash report. - `debug` - Sets the Sentry client into debug mode. - `tmp_path` - Filesystem path for temporary crash report state. diff --git a/src/Common/config_version.h.in b/src/Common/config_version.h.in index bc90e63e39c..c3c0c6df87b 100644 --- a/src/Common/config_version.h.in +++ b/src/Common/config_version.h.in @@ -20,6 +20,7 @@ #cmakedefine VERSION_MINOR @VERSION_MINOR@ #cmakedefine VERSION_PATCH @VERSION_PATCH@ #cmakedefine VERSION_STRING "@VERSION_STRING@" +#cmakedefine VERSION_STRING_SHORT "@VERSION_STRING_SHORT@" #cmakedefine VERSION_OFFICIAL "@VERSION_OFFICIAL@" #cmakedefine VERSION_FULL "@VERSION_FULL@" #cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@" From d154415a5bedf24a0217306c1d7798b718a2995a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 11:13:04 +0300 Subject: [PATCH 16/82] adjust comments --- .../en/operations/server-configuration-parameters/settings.md | 4 +++- programs/server/config.xml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index ba8f3df9ad0..3dc68e7fa6a 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -351,7 +351,9 @@ Keys: ## send_crash_reports {#server_configuration_parameters-logger} Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). -Enabling it, especially in pre-production environments, is strongly appreciated. +Enabling it, especially in pre-production environments, is greatly appreciated. + +The server will need an access to public Internet for this feature to be functioning properly. Keys: diff --git a/programs/server/config.xml b/programs/server/config.xml index 6086fcd7b1d..d8d75222bc0 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -46,6 +46,8 @@ false + + false From 52f7b9545b17304aa8e23373b77ab620fb338d50 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 11:24:21 +0300 Subject: [PATCH 17/82] Add http_proxy option --- base/daemon/SentryWriter.cpp | 6 ++++++ .../operations/server-configuration-parameters/settings.md | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 878ce6548aa..b2b1c69af8c 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -76,6 +76,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) sentry_options_set_environment(options, "test"); } + const std::string & http_proxy = config.getString("send_crash_reports.http_proxy", ""); + if (!http_proxy.empty()) + { + sentry_options_set_http_proxy(options, http_proxy.c_str()); + } + int init_status = sentry_init(options); if (!init_status) { diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 3dc68e7fa6a..194293d5a19 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -359,7 +359,8 @@ Keys: - `enabled` – Boolean flag to enable the feature. Set to `true` to allow sending crash reports. - `endpoint` – Overrides the Sentry endpoint. -- `anonymize` - Avoid attaching the server hostname to crash report. +- `anonymize` - Avoid attaching the server hostname to crash report. +- `http_proxy` - Configure HTTP proxy for sending crash reports. - `debug` - Sets the Sentry client into debug mode. - `tmp_path` - Filesystem path for temporary crash report state. From be94d8454dcefa117d43a8d96a01e4164be4ea51 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 30 May 2020 13:54:57 +0300 Subject: [PATCH 18/82] fix Arcadia build --- base/daemon/SentryWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index b2b1c69af8c..2fd846b720a 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -11,7 +11,7 @@ #endif #if USE_SENTRY -# include +# include // Y_IGNORE #endif From fd3279f9f16e704d78a0fd425216679651bc45ac Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 00:02:08 +0300 Subject: [PATCH 19/82] trigger ci --- docs/en/operations/server-configuration-parameters/settings.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 194293d5a19..a3a6d1a0955 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -364,7 +364,6 @@ Keys: - `debug` - Sets the Sentry client into debug mode. - `tmp_path` - Filesystem path for temporary crash report state. - **Recommended way to use** ``` xml From 81989bd95a91f0e1d70fb49bcfb4167ecbdd59c1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 10:51:22 +0300 Subject: [PATCH 20/82] submodule via https --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index daa5d12a62c..4175eb223db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = git@github.com:getsentry/sentry-native.git + url = https://github.com/getsentry/sentry-native.git From ba112e84cb10891cfdfa561ef6da4fd40693693e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 13:30:11 +0300 Subject: [PATCH 21/82] trigger ci From 3e0811f297cfdc77c657aa0a8ce04eb387e55ec6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 17:15:14 +0300 Subject: [PATCH 22/82] Adapt to recent logging changes --- base/daemon/SentryWriter.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 2fd846b720a..f7edc8d1e93 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -44,6 +44,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) #if USE_SENTRY bool enabled = false; bool debug = config.getBool("send_crash_reports.debug", false); + auto logger = &Poco::Logger::get("SentryWriter"); if (config.getBool("send_crash_reports.enabled", false)) { if (debug || (strlen(VERSION_OFFICIAL) > 0)) @@ -89,7 +90,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) anonymize = config.getBool("send_crash_reports.anonymize", false); const std::string& anonymize_status = anonymize ? " (anonymized)" : ""; LOG_INFO( - &Logger::get("SentryWriter"), + logger, "Sending crash reports is initialized with {} endpoint and {} temp folder{}", endpoint, temp_folder_path, @@ -97,12 +98,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) } else { - LOG_WARNING(&Logger::get("SentryWriter"), "Sending crash reports failed to initialized with {} status", init_status); + LOG_WARNING(logger, "Sending crash reports failed to initialized with {} status", init_status); } } else { - LOG_INFO(&Logger::get("SentryWriter"), "Sending crash reports is disabled"); + LOG_INFO(logger, "Sending crash reports is disabled"); } #endif } @@ -120,6 +121,7 @@ void SentryWriter::shutdown() void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace) { #if USE_SENTRY + auto logger = &Poco::Logger::get("SentryWriter"); if (initialized) { const std::string & error_message = signalToErrorMessage(sig, info, context); @@ -181,13 +183,13 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c sentry_value_set_by_key(event, "threads", threads); - LOG_INFO(&Logger::get("SentryWriter"), "Sending crash report"); + LOG_INFO(logger, "Sending crash report"); sentry_capture_event(event); shutdown(); } else { - LOG_INFO(&Logger::get("SentryWriter"), "Not sending crash report"); + LOG_INFO(logger, "Not sending crash report"); } #endif } From 9ad1bb8d9398cb83a95b39df8d36a17d340afc16 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 18:26:20 +0300 Subject: [PATCH 23/82] trigger ci From acf22bfb19292c5ae56e54dcedd06895577e2914 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 19:42:59 +0300 Subject: [PATCH 24/82] fix sanitizers build --- base/daemon/BaseDaemon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 3aeebd369e5..9da8849342d 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -317,7 +317,7 @@ static void sanitizerDeathCallback() std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) - bare_stacktrace << ' ' << stack_trace.getFrames()[i]; + bare_stacktrace << ' ' << stack_trace.getFramePointers()[i]; LOG_FATAL(log, bare_stacktrace.str()); } From 1ce25238f80fc9435c82766f44da896639e97ee1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 19:49:11 +0300 Subject: [PATCH 25/82] try fix some more build issues --- cmake/find/sentry.cmake | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index f94b53ffb00..30b8b28f6f1 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,15 +5,17 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -option (USE_SENTRY "Use Sentry" ON) +if (NOT OS_FREEBSD AND NOT UNBUNDLED) + option (USE_SENTRY "Use Sentry" ON) -set (BUILD_SHARED_LIBS OFF) -set (SENTRY_PIC OFF) -set (SENTRY_BACKEND "none") -set (SENTRY_TRANSPORT "curl") -set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) -set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) + set (BUILD_SHARED_LIBS OFF) + set (SENTRY_PIC OFF) + set (SENTRY_BACKEND "none") + set (SENTRY_TRANSPORT "curl") + set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) + set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) -message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") + message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") -include_directories("${SENTRY_INCLUDE_DIR}") \ No newline at end of file + include_directories("${SENTRY_INCLUDE_DIR}") +endif () \ No newline at end of file From f6e69355faa1f131fd22bfca93bc1cee0c1aca1e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 21:10:19 +0300 Subject: [PATCH 26/82] experiment --- src/Common/StackTrace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index aa78ab62f9b..526edd7792f 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -3,8 +3,8 @@ #include #include #include -#include #include +#include #include #include From 1797a47a9f26d4a97e8aa044af5a45a6c34e6d4f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 21:25:25 +0300 Subject: [PATCH 27/82] fix clang warnings --- base/daemon/SentryWriter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index f7edc8d1e93..6bfc07ea2fb 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -17,8 +17,8 @@ namespace { -static bool initialized = false; -static bool anonymize = false; +bool initialized = false; +bool anonymize = false; void setExtras() { @@ -44,7 +44,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) #if USE_SENTRY bool enabled = false; bool debug = config.getBool("send_crash_reports.debug", false); - auto logger = &Poco::Logger::get("SentryWriter"); + auto * logger = &Poco::Logger::get("SentryWriter"); if (config.getBool("send_crash_reports.enabled", false)) { if (debug || (strlen(VERSION_OFFICIAL) > 0)) @@ -121,7 +121,7 @@ void SentryWriter::shutdown() void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace) { #if USE_SENTRY - auto logger = &Poco::Logger::get("SentryWriter"); + auto * logger = &Poco::Logger::get("SentryWriter"); if (initialized) { const std::string & error_message = signalToErrorMessage(sig, info, context); From 8babd4d18c093f44d96e616a31d5551b24e73de2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 21:36:33 +0300 Subject: [PATCH 28/82] experiment --- cmake/find/sentry.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 30b8b28f6f1..06312b64495 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -6,6 +6,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") endif () if (NOT OS_FREEBSD AND NOT UNBUNDLED) + cmake_policy (SET CMP0077 NEW) option (USE_SENTRY "Use Sentry" ON) set (BUILD_SHARED_LIBS OFF) From 965204dfb161953616b6fa5168bde03375d87205 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 21:48:34 +0300 Subject: [PATCH 29/82] Try to fix the msan build --- .gitmodules | 2 +- contrib/sentry-native | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4175eb223db..ff4e644f657 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/getsentry/sentry-native.git + url = https://github.com/blinkov/sentry-native.git diff --git a/contrib/sentry-native b/contrib/sentry-native index 3bfce2d17c1..9e214a1265a 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit 3bfce2d17c1b80fbbaae83bb5ef41c1b290d34fb +Subproject commit 9e214a1265a4ea628c21045b7f43d1aec15e385d From 65ff11aeac99061b53de62ab120d9ff75ae0dc03 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 22:49:00 +0300 Subject: [PATCH 30/82] old cmake compatibility --- cmake/find/sentry.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 06312b64495..d10c15cd334 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -6,7 +6,9 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") endif () if (NOT OS_FREEBSD AND NOT UNBUNDLED) - cmake_policy (SET CMP0077 NEW) + if (POLICY CMP0077) + cmake_policy (SET CMP0077 NEW) + endif () option (USE_SENTRY "Use Sentry" ON) set (BUILD_SHARED_LIBS OFF) From e9a04f7741550a48d6c963fe4a225bf3ce616141 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 23:48:42 +0300 Subject: [PATCH 31/82] more build fixes --- base/daemon/SentryWriter.cpp | 4 ++-- src/Common/StackTrace.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 6bfc07ea2fb..95189b72e81 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -145,8 +145,8 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c { const StackTrace::Frame & current_frame = stack_trace.getFrames().value()[i]; sentry_value_t frame = sentry_value_new_object(); - unsigned long long frame_ptr = reinterpret_cast(current_frame.virtual_addr); - snprintf(instruction_addr, sizeof(instruction_addr), "0x%llx", frame_ptr); + UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); + std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIu64 "x", frame_ptr); sentry_value_set_by_key(frame, "instruction_addr", sentry_value_new_string(instruction_addr)); if (current_frame.symbol.has_value()) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 526edd7792f..e71ce1e1139 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -192,7 +192,7 @@ static void * getCallerAddress(const ucontext_t & context) static void symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) { -#if defined(__ELF__) && !defined(__FreeBSD__) +#if defined(__ELF__) && !defined(__FreeBSD__) && !defined(ARCADIA_BUILD) const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); std::unordered_map dwarfs; From 5a32d7913524a139ce43b835f53f73e1f0b42943 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 1 Jun 2020 23:55:32 +0300 Subject: [PATCH 32/82] experiment --- cmake/find/sentry.cmake | 2 +- src/Common/StackTrace.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index d10c15cd334..309f63e9165 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,7 +5,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD AND NOT UNBUNDLED) +if (NOT OS_FREEBSD AND NOT UNBUNDLED AND NOT SPLITTED AND NOT (COMPILER_CLANG AND OS_DARWIN)) if (POLICY CMP0077) cmake_policy (SET CMP0077 NEW) endif () diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index e71ce1e1139..dbe3d005be7 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -313,7 +313,7 @@ const StackTrace::Frames & StackTrace::getFrames() const { if (!frames.has_value()) { - frames = {{}}; + frames = std::array(); symbolize(frame_pointers.data(), offset, size, frames); } return frames; From 35734aadde1b7cd0f68cc6a513bab1e8497229f8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 08:15:11 +0300 Subject: [PATCH 33/82] apply comment --- src/Common/StackTrace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index dbe3d005be7..793de7709cc 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -313,7 +313,7 @@ const StackTrace::Frames & StackTrace::getFrames() const { if (!frames.has_value()) { - frames = std::array(); + frames.emplace({}); symbolize(frame_pointers.data(), offset, size, frames); } return frames; From 03f4aa19aa64e10e8433e938931e95400db7fdf6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 08:16:41 +0300 Subject: [PATCH 34/82] apply comment --- src/Common/StackTrace.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 793de7709cc..819f74f37cb 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -313,7 +313,7 @@ const StackTrace::Frames & StackTrace::getFrames() const { if (!frames.has_value()) { - frames.emplace({}); + frames.emplace(); symbolize(frame_pointers.data(), offset, size, frames); } return frames; @@ -357,7 +357,7 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs { std::stringstream out; StackTrace::Frames frames{}; - frames = {{}}; + frames.emplace(); symbolize(frame_pointers, offset, size, frames); toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); From 40f6e559e2d2424e29604d5191dee6b941bc3d6e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 08:29:13 +0300 Subject: [PATCH 35/82] fix compiling when disabled --- base/daemon/SentryWriter.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 95189b72e81..15602be2581 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #if !defined(ARCADIA_BUILD) @@ -105,6 +106,8 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { LOG_INFO(logger, "Sending crash reports is disabled"); } +#else + UNUSED(config); #endif } @@ -191,5 +194,10 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c { LOG_INFO(logger, "Not sending crash report"); } +#else + UNUSED(sig); + UNUSED(info); + UNUSED(context); + UNUSED(stack_trace); #endif } From 9c1ac2f1c1af35b20bb7ea031370a3c8e347f4df Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 09:46:36 +0300 Subject: [PATCH 36/82] experiment --- cmake/find/sentry.cmake | 9 +-------- contrib/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 309f63e9165..94c4f4a6e93 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,16 +5,9 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD AND NOT UNBUNDLED AND NOT SPLITTED AND NOT (COMPILER_CLANG AND OS_DARWIN)) - if (POLICY CMP0077) - cmake_policy (SET CMP0077 NEW) - endif () +if (NOT OS_FREEBSD) option (USE_SENTRY "Use Sentry" ON) - set (BUILD_SHARED_LIBS OFF) - set (SENTRY_PIC OFF) - set (SENTRY_BACKEND "none") - set (SENTRY_TRANSPORT "curl") set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ea13969db16..d9af4bc0ac5 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -321,6 +321,8 @@ if (USE_FASTOPS) endif() if (USE_SENTRY) + set (SENTRY_BACKEND "none") + set (SENTRY_TRANSPORT "curl") add_subdirectory (sentry-native) endif() From 280eea1e12fa4770f114ad952efa0be0eecc3e34 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 10:33:11 +0300 Subject: [PATCH 37/82] fix compiling when disabled --- base/daemon/SentryWriter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 15602be2581..c8197d8a160 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -16,14 +16,16 @@ #endif +#if USE_SENTRY namespace { + bool initialized = false; bool anonymize = false; void setExtras() { -#if USE_SENTRY + if (!anonymize) { sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); @@ -36,9 +38,9 @@ void setExtras() sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR)); sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); +} +} #endif -} -} void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { From 0e8d559d832df40c23942673aac254728b0e77b1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 13:13:21 +0300 Subject: [PATCH 38/82] disable for splitted --- cmake/find/sentry.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 94c4f4a6e93..4a5fe6f2478 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,7 +5,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD) +if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES) option (USE_SENTRY "Use Sentry" ON) set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) From 5036ad7c6afad995da38bf38de76f1e7134a4137 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 13:13:29 +0300 Subject: [PATCH 39/82] back to upstream --- .gitmodules | 2 +- contrib/sentry-native | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ff4e644f657..4175eb223db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/blinkov/sentry-native.git + url = https://github.com/getsentry/sentry-native.git diff --git a/contrib/sentry-native b/contrib/sentry-native index 9e214a1265a..aed9c18536d 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit 9e214a1265a4ea628c21045b7f43d1aec15e385d +Subproject commit aed9c18536dff1851b1240f84263a55ef716acb6 From 862693d78dc1924f472646584fbd70f955239c0c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 16:59:45 +0300 Subject: [PATCH 40/82] change sentry-native commit --- contrib/sentry-native | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/sentry-native b/contrib/sentry-native index aed9c18536d..b48c21d2440 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit aed9c18536dff1851b1240f84263a55ef716acb6 +Subproject commit b48c21d244092658d6e2d1bb243b705fd968b9f7 From 711e7d101da19c8364b761ee09ca042bf9c680f8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 21:50:55 +0300 Subject: [PATCH 41/82] experiment --- base/CMakeLists.txt | 5 +++++ base/daemon/CMakeLists.txt | 10 +++------- cmake/find/sentry.cmake | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index cfa54fe2ca4..ad3bf56cd00 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -11,3 +11,8 @@ add_subdirectory (widechar_width) if (USE_MYSQL) add_subdirectory (mysqlxx) endif () + +if (USE_SENTRY) + target_link_libraries (daemon PRIVATE curl) + target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) +endif () \ No newline at end of file diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 0b6a7188c83..8f70f30aeb1 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -1,12 +1,8 @@ add_library (daemon BaseDaemon.cpp GraphiteWriter.cpp - SentryWriter.cpp) + SentryWriter.cpp +) target_include_directories (daemon PUBLIC ..) -target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) - -if (USE_SENTRY) - target_link_libraries (daemon PRIVATE curl) - target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) -endif () \ No newline at end of file +target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) \ No newline at end of file diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 4a5fe6f2478..449d995935d 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -8,6 +8,8 @@ endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES) option (USE_SENTRY "Use Sentry" ON) + set (SENTRY_TRANSPORT "url") + set (SENTRY_BACKEND "none") set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) From 921b7c748000cc4a33c9db618716922fc34f1f17 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 22:25:34 +0300 Subject: [PATCH 42/82] partial revert --- base/CMakeLists.txt | 5 ----- base/daemon/CMakeLists.txt | 7 ++++++- cmake/find/sentry.cmake | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index ad3bf56cd00..a8dedec9269 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -10,9 +10,4 @@ add_subdirectory (widechar_width) if (USE_MYSQL) add_subdirectory (mysqlxx) -endif () - -if (USE_SENTRY) - target_link_libraries (daemon PRIVATE curl) - target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) endif () \ No newline at end of file diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 8f70f30aeb1..36de193bccd 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -5,4 +5,9 @@ add_library (daemon ) target_include_directories (daemon PUBLIC ..) -target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) \ No newline at end of file +target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) + +if (USE_SENTRY) + target_link_libraries (daemon PRIVATE curl) + target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) +endif () \ No newline at end of file diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 449d995935d..6848dc00b43 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,7 +5,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES) +if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG)) option (USE_SENTRY "Use Sentry" ON) set (SENTRY_TRANSPORT "url") From 2f74c58b0598a14db9583b660f2316b01013f052 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 2 Jun 2020 23:50:18 +0300 Subject: [PATCH 43/82] experiment with BUILD_SHARED_LIBS --- cmake/find/sentry.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 6848dc00b43..08f712d5574 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -12,8 +12,11 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILE set (SENTRY_BACKEND "none") set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) + if (NOT_UNBUNDLED) + set (BUILD_SHARED_LIBS OFF) + endif() message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") -endif () \ No newline at end of file +endif () From 6f0e754f1e6bfd5753c04b1f81b231eece4f82fa Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 4 Jun 2020 11:57:01 +0300 Subject: [PATCH 44/82] try to fix the glibc compatibility --- .gitmodules | 2 +- cmake/find/sentry.cmake | 9 ++++----- contrib/CMakeLists.txt | 2 -- contrib/sentry-native | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4175eb223db..ff4e644f657 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/getsentry/sentry-native.git + url = https://github.com/blinkov/sentry-native.git diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 08f712d5574..e1cd28c1d59 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -7,15 +7,14 @@ endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG)) option (USE_SENTRY "Use Sentry" ON) - - set (SENTRY_TRANSPORT "url") - set (SENTRY_BACKEND "none") set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) - if (NOT_UNBUNDLED) + set (SENTRY_TRANSPORT "curl" CACHE STRING "") + set (SENTRY_BACKEND "none" CACHE STRING "") + set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") + if (OS_LINUX AND NOT_UNBUNDLED) set (BUILD_SHARED_LIBS OFF) endif() - message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d9af4bc0ac5..ea13969db16 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -321,8 +321,6 @@ if (USE_FASTOPS) endif() if (USE_SENTRY) - set (SENTRY_BACKEND "none") - set (SENTRY_TRANSPORT "curl") add_subdirectory (sentry-native) endif() diff --git a/contrib/sentry-native b/contrib/sentry-native index b48c21d2440..18835dd8c49 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit b48c21d244092658d6e2d1bb243b705fd968b9f7 +Subproject commit 18835dd8c496f22859bd6a1a7054a2bd4762e7ed From 966593e0a8e42266138cbd5af917ec0be60a3c0a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 9 Jun 2020 15:50:18 +0300 Subject: [PATCH 45/82] try to completely remove sentry from odbc-bridge --- programs/odbc-bridge/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index ab8d94f2a0c..51abf4a9adb 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -10,7 +10,7 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES PingHandler.cpp validateODBCConnectionString.cpp ) - +set (USE_SENTRY OFF CACHE BOOL "" FORCE) set (CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE clickhouse_parsers From d91f0bd580aa8632cc89aae0531281f300b48740 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 9 Jun 2020 19:07:40 +0300 Subject: [PATCH 46/82] Switch back to sentry upstream --- .gitmodules | 2 +- contrib/sentry-native | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ff4e644f657..4175eb223db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/blinkov/sentry-native.git + url = https://github.com/getsentry/sentry-native.git diff --git a/contrib/sentry-native b/contrib/sentry-native index 18835dd8c49..9651561d45e 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit 18835dd8c496f22859bd6a1a7054a2bd4762e7ed +Subproject commit 9651561d45e4d00e9fe708275c086a3cfeb496bd From f872c639ed6893d0731ed61c1927f3c6f313f0d2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 9 Jun 2020 20:44:56 +0300 Subject: [PATCH 47/82] Try to disable linker options from sentry --- .gitmodules | 2 +- cmake/find/sentry.cmake | 1 + contrib/sentry-native | 2 +- programs/odbc-bridge/CMakeLists.txt | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4175eb223db..ff4e644f657 100644 --- a/.gitmodules +++ b/.gitmodules @@ -162,4 +162,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/getsentry/sentry-native.git + url = https://github.com/blinkov/sentry-native.git diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index e1cd28c1d59..7fa384cb906 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -11,6 +11,7 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILE set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) set (SENTRY_TRANSPORT "curl" CACHE STRING "") set (SENTRY_BACKEND "none" CACHE STRING "") + set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") if (OS_LINUX AND NOT_UNBUNDLED) set (BUILD_SHARED_LIBS OFF) diff --git a/contrib/sentry-native b/contrib/sentry-native index 9651561d45e..78fb54989cd 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit 9651561d45e4d00e9fe708275c086a3cfeb496bd +Subproject commit 78fb54989cd61cf11dcea142e12d1ecc6940c962 diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 51abf4a9adb..af59383d030 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -10,7 +10,6 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES PingHandler.cpp validateODBCConnectionString.cpp ) -set (USE_SENTRY OFF CACHE BOOL "" FORCE) set (CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE clickhouse_parsers From 6191d33bd9cb67d4dcabb79202bdd91baf467ddd Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 16:30:12 +0300 Subject: [PATCH 48/82] Do not cache frames inside StackTrace --- base/daemon/BaseDaemon.cpp | 1 - cmake/find/sentry.cmake | 1 + src/Common/StackTrace.cpp | 30 ++++++++---------------------- src/Common/StackTrace.h | 7 +------ 4 files changed, 10 insertions(+), 29 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 9da8849342d..1467657d31a 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -223,7 +223,6 @@ public: DB::readPODBinary(stack_trace, in); DB::readBinary(thread_num, in); DB::readBinary(query_id, in); - stack_trace.resetFrames(); /// This allows to receive more signals if failure happens inside onFault function. /// Example: segfault while symbolizing stack trace. diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 7fa384cb906..2281d870dec 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -13,6 +13,7 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILE set (SENTRY_BACKEND "none" CACHE STRING "") set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") + set (SENTRY_PIC OFF CACHE BOOL "") if (OS_LINUX AND NOT_UNBUNDLED) set (BUILD_SHARED_LIBS OFF) endif() diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 819f74f37cb..aacda116bfb 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -199,12 +199,12 @@ static void symbolize(const void * const * frame_pointers, size_t offset, size_t for (size_t i = 0; i < offset; ++i) { - frames.value()[i].virtual_addr = frame_pointers[i]; + frames[i].virtual_addr = frame_pointers[i]; } for (size_t i = offset; i < size; ++i) { - StackTrace::Frame & current_frame = frames.value()[i]; + StackTrace::Frame & current_frame = frames[i]; current_frame.virtual_addr = frame_pointers[i]; const auto * object = symbol_index.findObject(current_frame.virtual_addr); uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; @@ -244,7 +244,7 @@ static void symbolize(const void * const * frame_pointers, size_t offset, size_t #else for (size_t i = 0; i < size; ++i) { - frames.value()[i].virtual_addr = frame_pointers[i]; + frames[i].virtual_addr = frame_pointers[i]; } UNUSED(offset); #endif @@ -309,16 +309,6 @@ const StackTrace::FramePointers & StackTrace::getFramePointers() const return frame_pointers; } -const StackTrace::Frames & StackTrace::getFrames() const -{ - if (!frames.has_value()) - { - frames.emplace(); - symbolize(frame_pointers.data(), offset, size, frames); - } - return frames; -} - static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t size, std::function callback) { @@ -329,7 +319,7 @@ toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t s for (size_t i = offset; i < size; ++i) { - const StackTrace::Frame & current_frame = frames.value()[i]; + const StackTrace::Frame & current_frame = frames[i]; out << i << ". "; if (current_frame.file.has_value() && current_frame.line.has_value()) @@ -356,8 +346,7 @@ toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t s static std::string toStringImpl(const void * const * frame_pointers, size_t offset, size_t size) { std::stringstream out; - StackTrace::Frames frames{}; - frames.emplace(); + StackTrace::Frames frames; symbolize(frame_pointers, offset, size, frames); toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); @@ -365,12 +354,9 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs void StackTrace::toStringEveryLine(std::function callback) const { - toStringEveryLineImpl(getFrames(), offset, size, std::move(callback)); -} - -void StackTrace::resetFrames() -{ - frames.reset(); + Frames frames; + symbolize(frame_pointers.data(), offset, size, frames); + toStringEveryLineImpl(frames, offset, size, std::move(callback)); } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 27b2c44dd94..4ec63b3cf86 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -36,7 +36,7 @@ public: }; static constexpr size_t capacity = 32; using FramePointers = std::array; - using Frames = std::optional>; + using Frames = std::array; /// Tries to capture stack trace StackTrace(); @@ -51,22 +51,17 @@ public: size_t getSize() const; size_t getOffset() const; const FramePointers & getFramePointers() const; - const Frames & getFrames() const; std::string toString() const; static std::string toString(void ** frame_pointers, size_t offset, size_t size); void toStringEveryLine(std::function callback) const; - - void resetFrames(); - protected: void tryCapture(); size_t size = 0; size_t offset = 0; /// How many frames to skip while displaying. FramePointers frame_pointers{}; - mutable Frames frames{}; }; std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context); From 60b40f04039702a0d8d55c44cfb81e96c932836e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 17:51:25 +0300 Subject: [PATCH 49/82] Lost part of refactoring --- base/daemon/SentryWriter.cpp | 4 +++- src/Common/StackTrace.cpp | 6 +++--- src/Common/StackTrace.h | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index c8197d8a160..eddd5bfa49c 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -146,9 +146,11 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c offset = 1; } char instruction_addr[100]; + StackTrace::Frames frames; + StackTrace::symbolize(stack_trace.getFramePointers().data(), offset, size, frames); for (size_t i = stack_size - 1; i >= offset; --i) { - const StackTrace::Frame & current_frame = stack_trace.getFrames().value()[i]; + const StackTrace::Frame & current_frame = frames[i]; sentry_value_t frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIu64 "x", frame_ptr); diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index aacda116bfb..8e390154838 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -190,7 +190,7 @@ static void * getCallerAddress(const ucontext_t & context) #endif } -static void symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) +void StackTrace::symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) { #if defined(__ELF__) && !defined(__FreeBSD__) && !defined(ARCADIA_BUILD) @@ -347,7 +347,7 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs { std::stringstream out; StackTrace::Frames frames; - symbolize(frame_pointers, offset, size, frames); + StackTrace::symbolize(frame_pointers.data(), offset, size, frames); toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); } @@ -355,7 +355,7 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs void StackTrace::toStringEveryLine(std::function callback) const { Frames frames; - symbolize(frame_pointers.data(), offset, size, frames); + StackTrace::symbolize(frame_pointers.data(), offset, size, frames); toStringEveryLineImpl(frames, offset, size, std::move(callback)); } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 4ec63b3cf86..374f0314533 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -54,6 +54,7 @@ public: std::string toString() const; static std::string toString(void ** frame_pointers, size_t offset, size_t size); + static void symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); void toStringEveryLine(std::function callback) const; protected: From 0316464ed4eb22593bd7fc18b79584cc0f476ce0 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 18:30:13 +0300 Subject: [PATCH 50/82] fix --- src/Common/StackTrace.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 8e390154838..cb0488b489a 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -347,7 +347,7 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs { std::stringstream out; StackTrace::Frames frames; - StackTrace::symbolize(frame_pointers.data(), offset, size, frames); + StackTrace::symbolize(frame_pointers, offset, size, frames); toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); } From 0a74c9373ec46713b3deb51e86c702418fc29a0d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 19:48:08 +0300 Subject: [PATCH 51/82] less confusing --- base/daemon/SentryWriter.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index eddd5bfa49c..003a2816ce0 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -136,7 +136,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c setExtras(); /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/stacktrace/ - sentry_value_t frames = sentry_value_new_list(); + sentry_value_t sentry_frames = sentry_value_new_list(); size_t stack_size = stack_trace.getSize(); if (stack_size > 0) { @@ -151,33 +151,33 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c for (size_t i = stack_size - 1; i >= offset; --i) { const StackTrace::Frame & current_frame = frames[i]; - sentry_value_t frame = sentry_value_new_object(); + sentry_value_t sentry_frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIu64 "x", frame_ptr); - sentry_value_set_by_key(frame, "instruction_addr", sentry_value_new_string(instruction_addr)); + sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); if (current_frame.symbol.has_value()) { - sentry_value_set_by_key(frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); + sentry_value_set_by_key(sentry_frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); } if (current_frame.file.has_value()) { - sentry_value_set_by_key(frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); + sentry_value_set_by_key(sentry_frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); } if (current_frame.line.has_value()) { - sentry_value_set_by_key(frame, "lineno", sentry_value_new_int32(current_frame.line.value())); + sentry_value_set_by_key(sentry_frame, "lineno", sentry_value_new_int32(current_frame.line.value())); } - sentry_value_append(frames, frame); + sentry_value_append(sentry_frames, sentry_frame); } } /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/threads/ sentry_value_t stacktrace = sentry_value_new_object(); - sentry_value_set_by_key(stacktrace, "frames", frames); + sentry_value_set_by_key(stacktrace, "frames", sentry_frames); sentry_value_t thread = sentry_value_new_object(); sentry_value_set_by_key(thread, "stacktrace", stacktrace); From 5fa44019918581a4378582bcd72ff5b160e5f659 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 21:18:34 +0300 Subject: [PATCH 52/82] fix --- base/daemon/SentryWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 003a2816ce0..2ce43c9f0a2 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -147,7 +147,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c } char instruction_addr[100]; StackTrace::Frames frames; - StackTrace::symbolize(stack_trace.getFramePointers().data(), offset, size, frames); + StackTrace::symbolize(stack_trace.getFramePointers().data(), offset, stack_size, frames); for (size_t i = stack_size - 1; i >= offset; --i) { const StackTrace::Frame & current_frame = frames[i]; From 67ccd6703ea9de805b65c7fa25a3c43620571b55 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 00:03:13 +0300 Subject: [PATCH 53/82] maybe fix the unbundled gcc build --- cmake/find/sentry.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 2281d870dec..2d3aa71248a 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,7 +5,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG)) +if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) option (USE_SENTRY "Use Sentry" ON) set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) @@ -14,9 +14,7 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILE set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") set (SENTRY_PIC OFF CACHE BOOL "") - if (OS_LINUX AND NOT_UNBUNDLED) - set (BUILD_SHARED_LIBS OFF) - endif() + set (BUILD_SHARED_LIBS OFF) message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") From 22707508c1ccb85d3dd9e4fd9bbee39b73ade962 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 12:17:33 +0300 Subject: [PATCH 54/82] experiment --- tests/queries/0_stateless/00816_long_concurrent_alter_column.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 965408065cf..d3a26b0ed75 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -59,6 +59,7 @@ wait echo "DROP TABLE concurrent_alter_column" | ${CLICKHOUSE_CLIENT} +sleep 1 # Check for deadlocks echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter%'" | ${CLICKHOUSE_CLIENT} From 706c5452482d633b69b8fa54c0eb6b3ccd248d9e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 12:18:12 +0300 Subject: [PATCH 55/82] experiment --- tests/queries/0_stateless/00816_long_concurrent_alter_column.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index d3a26b0ed75..3ed0c6e1a6a 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -59,7 +59,7 @@ wait echo "DROP TABLE concurrent_alter_column" | ${CLICKHOUSE_CLIENT} -sleep 1 +sleep 7 # Check for deadlocks echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter%'" | ${CLICKHOUSE_CLIENT} From 3e5d735871c9a995c5450b05d030d3046f2d3051 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 12:21:23 +0300 Subject: [PATCH 56/82] back to upstream --- .gitmodules | 2 +- contrib/sentry-native | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 93a0078a051..2fed57a519d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -170,4 +170,4 @@ url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/blinkov/sentry-native.git + url = https://github.com/getsentry/sentry-native.git diff --git a/contrib/sentry-native b/contrib/sentry-native index 78fb54989cd..f91ed3f95b5 160000 --- a/contrib/sentry-native +++ b/contrib/sentry-native @@ -1 +1 @@ -Subproject commit 78fb54989cd61cf11dcea142e12d1ecc6940c962 +Subproject commit f91ed3f95b5653f247189d720ab00765b4899d6f From 5f73c87c7142e7d137a29f827d8fab8ebdc10ad2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 15:18:19 +0300 Subject: [PATCH 57/82] change used flag --- cmake/find/sentry.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 2d3aa71248a..eadf071141e 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -5,7 +5,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") return() endif () -if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) +if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) option (USE_SENTRY "Use Sentry" ON) set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) From fa47fc3f30eb144aa1593ebbfc630fdb4f095c39 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 15:34:02 +0300 Subject: [PATCH 58/82] fix address formatting --- base/daemon/SentryWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 2ce43c9f0a2..45f5bd56ca1 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -153,7 +153,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c const StackTrace::Frame & current_frame = frames[i]; sentry_value_t sentry_frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); - std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIu64 "x", frame_ptr); + std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIx64, frame_ptr); sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); if (current_frame.symbol.has_value()) From bbeb768a1952541a895b51ecb70eee5dd4532224 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 11 Jun 2020 21:12:48 +0300 Subject: [PATCH 59/82] use the sentry logger hook --- base/daemon/SentryWriter.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 45f5bd56ca1..bb176db813c 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -13,6 +13,7 @@ #if USE_SENTRY # include // Y_IGNORE +# include #endif @@ -39,6 +40,33 @@ void setExtras() sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR)); sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH)); } + +void sentry_logger(sentry_level_t level, const char * message, va_list args) +{ + auto * logger = &Poco::Logger::get("SentryWriter"); + size_t size = 1024; + char buffer[size]; + if (vsnprintf(buffer, size, message, args) >= 0) { + switch (level) { + case SENTRY_LEVEL_DEBUG: + logger->debug(buffer); + break; + case SENTRY_LEVEL_INFO: + logger->information(buffer); + break; + case SENTRY_LEVEL_WARNING: + logger->warning(buffer); + break; + case SENTRY_LEVEL_ERROR: + logger->error(buffer); + break; + case SENTRY_LEVEL_FATAL: + logger->fatal(buffer); + break; + } + } +} +} } #endif @@ -65,6 +93,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) sentry_options_t * options = sentry_options_new(); sentry_options_set_release(options, VERSION_STRING); + sentry_options_set_logger(options, &sentry_logger); if (debug) { sentry_options_set_debug(options, 1); From e92641858e6fdf132c4b1f9ecf401fc985d2693e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 12 Jun 2020 01:00:35 +0300 Subject: [PATCH 60/82] fixes --- base/daemon/SentryWriter.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index bb176db813c..d7f08864e96 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -46,8 +46,10 @@ void sentry_logger(sentry_level_t level, const char * message, va_list args) auto * logger = &Poco::Logger::get("SentryWriter"); size_t size = 1024; char buffer[size]; - if (vsnprintf(buffer, size, message, args) >= 0) { - switch (level) { + if (vsnprintf(buffer, size, message, args) >= 0) + { + switch (level) + { case SENTRY_LEVEL_DEBUG: logger->debug(buffer); break; @@ -67,7 +69,6 @@ void sentry_logger(sentry_level_t level, const char * message, va_list args) } } } -} #endif void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) From 395ef1ecafff89a880bd47c2b11aa12b42e52c03 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 12 Jun 2020 09:35:31 +0300 Subject: [PATCH 61/82] experiment --- base/daemon/SentryWriter.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index d7f08864e96..88639d8bf94 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -46,8 +46,15 @@ void sentry_logger(sentry_level_t level, const char * message, va_list args) auto * logger = &Poco::Logger::get("SentryWriter"); size_t size = 1024; char buffer[size]; +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif if (vsnprintf(buffer, size, message, args) >= 0) { +#ifdef __clang__ +#pragma clang diagnostic pop +#endif switch (level) { case SENTRY_LEVEL_DEBUG: From a9514d725768d0025ed848e2a5ea016fb8ac5001 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 12 Jun 2020 16:52:41 +0300 Subject: [PATCH 62/82] trigger ci From 62eaeac713a130f73c74e6efc6d831009a582a5e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 13 Jun 2020 05:35:42 +0300 Subject: [PATCH 63/82] trigger ci From c70f7778fc6144d22c69b3a1972b4720e3b6788e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sat, 13 Jun 2020 23:05:13 +0300 Subject: [PATCH 64/82] trigger ci From ad321966f09e8801b902dffa650b8beb57924454 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sun, 14 Jun 2020 09:43:40 +0300 Subject: [PATCH 65/82] trigger ci From b8611cf46cd2d6f15f2a6e678961c06c686fa9ed Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sun, 14 Jun 2020 18:05:15 +0300 Subject: [PATCH 66/82] experiment --- tests/queries/0_stateless/00600_replace_running_query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00600_replace_running_query.sh b/tests/queries/0_stateless/00600_replace_running_query.sh index 1331dd3c15b..75006cc56ce 100755 --- a/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/tests/queries/0_stateless/00600_replace_running_query.sh @@ -36,5 +36,5 @@ wait ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null -${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' wait +${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' From c30457a3edf98c61cd45b4c4269f3f5d4679e3ba Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sun, 14 Jun 2020 20:34:59 +0300 Subject: [PATCH 67/82] trigger ci From 977fd3e44fa08fe8427e04e75386894c785fba1f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Jun 2020 02:45:05 +0300 Subject: [PATCH 68/82] Update CMakeLists.txt --- base/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index a8dedec9269..cfa54fe2ca4 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -10,4 +10,4 @@ add_subdirectory (widechar_width) if (USE_MYSQL) add_subdirectory (mysqlxx) -endif () \ No newline at end of file +endif () From 92c7760c6e87f64dca55b78c12176c4f35b5de6c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Jun 2020 02:51:33 +0300 Subject: [PATCH 69/82] Update CMakeLists.txt --- base/daemon/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 36de193bccd..04d2f059b39 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -10,4 +10,4 @@ target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickh if (USE_SENTRY) target_link_libraries (daemon PRIVATE curl) target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) -endif () \ No newline at end of file +endif () From 9b734ffded4cbf4e929bcfa2bb545c6d3e67938a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Jun 2020 03:21:20 +0300 Subject: [PATCH 70/82] Update http_server.py --- tests/integration/test_send_crash_reports/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_send_crash_reports/http_server.py b/tests/integration/test_send_crash_reports/http_server.py index e3fa2e1cb57..74f0592504f 100644 --- a/tests/integration/test_send_crash_reports/http_server.py +++ b/tests/integration/test_send_crash_reports/http_server.py @@ -40,4 +40,4 @@ if __name__ == "__main__": try: httpd.serve_forever() finally: - httpd.server_close() \ No newline at end of file + httpd.server_close() From 0e77692a278b818d03bfd0987c7115802f89a648 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 16 Jun 2020 15:56:28 +0300 Subject: [PATCH 71/82] improvements after review comments --- base/daemon/SentryWriter.cpp | 32 ++++++++++--------- base/daemon/SentryWriter.h | 10 +++++- .../settings.md | 2 +- programs/server/config.xml | 3 +- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 88639d8bf94..0524285ea42 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -14,6 +14,7 @@ #if USE_SENTRY # include // Y_IGNORE # include +# include #endif @@ -31,7 +32,7 @@ void setExtras() { sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); } - sentry_set_tag("version", VERSION_STRING_SHORT); + sentry_set_tag("version", VERSION_STRING); sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); sentry_set_extra("version_integer", sentry_value_new_int32(VERSION_INTEGER)); @@ -93,14 +94,15 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) } if (enabled) { + const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry"; const std::string & endpoint = config.getString("send_crash_reports.endpoint", "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277"); const std::string & temp_folder_path - = config.getString("send_crash_reports.tmp_path", config.getString("tmp_path", Poco::Path::temp()) + "sentry/"); + = config.getString("send_crash_reports.tmp_path", default_tmp_path); Poco::File(temp_folder_path).createDirectories(); - sentry_options_t * options = sentry_options_new(); - sentry_options_set_release(options, VERSION_STRING); + sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown + sentry_options_set_release(options, VERSION_STRING_SHORT); sentry_options_set_logger(options, &sentry_logger); if (debug) { @@ -128,17 +130,16 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { initialized = true; anonymize = config.getBool("send_crash_reports.anonymize", false); - const std::string& anonymize_status = anonymize ? " (anonymized)" : ""; LOG_INFO( logger, "Sending crash reports is initialized with {} endpoint and {} temp folder{}", endpoint, temp_folder_path, - anonymize_status); + anonymize ? " (anonymized)" : ""); } else { - LOG_WARNING(logger, "Sending crash reports failed to initialized with {} status", init_status); + LOG_WARNING(logger, "Sending crash reports failed to initialize with {} status", init_status); } } else @@ -177,21 +178,20 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c size_t stack_size = stack_trace.getSize(); if (stack_size > 0) { - size_t offset = stack_trace.getOffset(); - if (stack_size == 1) - { - offset = 1; - } + ssize_t offset = stack_trace.getOffset(); char instruction_addr[100]; StackTrace::Frames frames; StackTrace::symbolize(stack_trace.getFramePointers().data(), offset, stack_size, frames); - for (size_t i = stack_size - 1; i >= offset; --i) + for (ssize_t i = stack_size - 1; i >= offset; --i) { const StackTrace::Frame & current_frame = frames[i]; sentry_value_t sentry_frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); - std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIx64, frame_ptr); - sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); + + if (std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIx64, frame_ptr) >= 0) + { + sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); + } if (current_frame.symbol.has_value()) { @@ -213,6 +213,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c } /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/threads/ + /// Stacktrace is filled only for a single thread that failed sentry_value_t stacktrace = sentry_value_new_object(); sentry_value_set_by_key(stacktrace, "frames", sentry_frames); @@ -225,6 +226,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c sentry_value_t threads = sentry_value_new_object(); sentry_value_set_by_key(threads, "values", values); + sentry_value_set_by_key(event, "threads", threads); LOG_INFO(logger, "Sending crash report"); diff --git a/base/daemon/SentryWriter.h b/base/daemon/SentryWriter.h index 0b3f1ddd2b7..655a4e93bfd 100644 --- a/base/daemon/SentryWriter.h +++ b/base/daemon/SentryWriter.h @@ -7,7 +7,13 @@ #include -/// Sends crash reports to ClickHouse core developer team via https://sentry.io +/// \brief Sends crash reports to ClickHouse core developer team via https://sentry.io +/// +/// This feature can enabled with "send_crash_reports.enabled" server setting, +/// in this case reports are sent only for official ClickHouse builds. +/// +/// It is possible to send those reports to your own sentry account or account of consulting company you hired +/// by overriding "send_crash_reports.endpoint" setting. "send_crash_reports.debug" setting will allow to do that for class SentryWriter { public: @@ -15,6 +21,8 @@ public: static void initialize(Poco::Util::LayeredConfiguration & config); static void shutdown(); + + /// Not signal safe and can't be called from a signal handler static void onFault( int sig, const siginfo_t & info, diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index e1ff3a872d1..58a02b8266a 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -353,7 +353,7 @@ Keys: Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). Enabling it, especially in pre-production environments, is greatly appreciated. -The server will need an access to public Internet for this feature to be functioning properly. +The server will need an access to public Internet via IPv4 (at the time of writing IPv6 is not supported by Sentry) for this feature to be functioning properly. Keys: diff --git a/programs/server/config.xml b/programs/server/config.xml index afb44989bbe..e5482a074a3 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -46,7 +46,8 @@ - + + false From 8b50e3450b93e436c5d205ad9e4905c735ad5291 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 16 Jun 2020 23:01:15 +0300 Subject: [PATCH 72/82] move the default endpoint to config --- base/daemon/SentryWriter.cpp | 2 +- programs/server/config.xml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 0524285ea42..d979c671290 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -96,7 +96,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) { const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry"; const std::string & endpoint - = config.getString("send_crash_reports.endpoint", "https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277"); + = config.getString("send_crash_reports.endpoint"); const std::string & temp_folder_path = config.getString("send_crash_reports.tmp_path", default_tmp_path); Poco::File(temp_folder_path).createDirectories(); diff --git a/programs/server/config.xml b/programs/server/config.xml index e5482a074a3..d1226e6f09c 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -52,6 +52,9 @@ false false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 From 102628ff099b65c2bd764b2ea66ef8bcd1c606f5 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 17 Jun 2020 10:54:06 +0300 Subject: [PATCH 73/82] remove extra line --- base/daemon/SentryWriter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index d979c671290..b59df1ba55c 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -226,7 +226,6 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c sentry_value_t threads = sentry_value_new_object(); sentry_value_set_by_key(threads, "values", values); - sentry_value_set_by_key(event, "threads", threads); LOG_INFO(logger, "Sending crash report"); From 85974dd699b53084bb1a140ddea1ec18494685c1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 19 Jun 2020 22:56:18 +0300 Subject: [PATCH 74/82] Update 00816_long_concurrent_alter_column.sh --- tests/queries/0_stateless/00816_long_concurrent_alter_column.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 3ed0c6e1a6a..965408065cf 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -59,7 +59,6 @@ wait echo "DROP TABLE concurrent_alter_column" | ${CLICKHOUSE_CLIENT} -sleep 7 # Check for deadlocks echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter%'" | ${CLICKHOUSE_CLIENT} From 7b1ab482b0dc2fba2e80b5a30fb777a0a6153f22 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 22 Jun 2020 10:22:13 +0300 Subject: [PATCH 75/82] restore immediate stacktrace output --- src/Common/StackTrace.cpp | 60 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index cb0488b489a..3fde6b9e86c 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -269,7 +269,6 @@ StackTrace::StackTrace(const ucontext_t & signal_context) else { /// Skip excessive stack frames that we have created while finding stack trace. - for (size_t i = 0; i < size; ++i) { if (frame_pointers[i] == caller_address) @@ -309,54 +308,77 @@ const StackTrace::FramePointers & StackTrace::getFramePointers() const return frame_pointers; } -static void -toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offset, size_t size, std::function callback) +static void toStringEveryLineImpl(const void * const * frame_pointers, size_t offset, size_t size, std::function callback) { if (size == 0) return callback(""); +#if defined(__ELF__) && !defined(__FreeBSD__) + const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); + std::unordered_map dwarfs; + std::stringstream out; for (size_t i = offset; i < size; ++i) { - const StackTrace::Frame & current_frame = frames[i]; + const void * virtual_addr = frame_pointers[i]; + const auto * object = symbol_index.findObject(virtual_addr); + uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; + const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); + out << i << ". "; - if (current_frame.file.has_value() && current_frame.line.has_value()) + if (object) { - out << current_frame.file.value() << ":" << current_frame.line.value() << ": "; + if (std::filesystem::exists(object->name)) + { + auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; + + DB::Dwarf::LocationInfo location; + if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) + out << location.file.toString() << ":" << location.line << ": "; + } } - if (current_frame.symbol.has_value()) + const auto * symbol = symbol_index.findSymbol(virtual_addr); + if (symbol) { - out << current_frame.symbol.value(); + int status = 0; + out << demangle(symbol->name, status); } + else + out << "?"; - out << " @ " << current_frame.physical_addr; - if (current_frame.object.has_value()) - { - out << " in " << current_frame.object.value(); - } + out << " @ " << physical_addr; + out << " in " << (object ? object->name : "?"); callback(out.str()); out.str({}); } +#else + std::stringstream out; + + for (size_t i = offset; i < size; ++i) + { + const void * addr = frame_pointers[i]; + out << i << ". " << addr; + + callback(out.str()); + out.str({}); + } +#endif } static std::string toStringImpl(const void * const * frame_pointers, size_t offset, size_t size) { std::stringstream out; - StackTrace::Frames frames; - StackTrace::symbolize(frame_pointers, offset, size, frames); - toStringEveryLineImpl(frames, offset, size, [&](const std::string & str) { out << str << '\n'; }); + toStringEveryLineImpl(frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); } void StackTrace::toStringEveryLine(std::function callback) const { - Frames frames; - StackTrace::symbolize(frame_pointers.data(), offset, size, frames); - toStringEveryLineImpl(frames, offset, size, std::move(callback)); + toStringEveryLineImpl(frame_pointers.data(), offset, size, std::move(callback)); } From 87d56d700f3ba9b6dc40067cf6de3c1a288f0756 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 00:15:54 +0300 Subject: [PATCH 76/82] restore old toStringImpl signature --- base/daemon/SentryWriter.cpp | 2 +- src/Common/StackTrace.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index b59df1ba55c..ab560f4d8a1 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -181,7 +181,7 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c ssize_t offset = stack_trace.getOffset(); char instruction_addr[100]; StackTrace::Frames frames; - StackTrace::symbolize(stack_trace.getFramePointers().data(), offset, stack_size, frames); + StackTrace::symbolize(stack_trace.getFramePointers(), offset, stack_size, frames); for (ssize_t i = stack_size - 1; i >= offset; --i) { const StackTrace::Frame & current_frame = frames[i]; diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 3fde6b9e86c..1ea5d100440 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -190,7 +190,7 @@ static void * getCallerAddress(const ucontext_t & context) #endif } -void StackTrace::symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) +void StackTrace::symbolize(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames) { #if defined(__ELF__) && !defined(__FreeBSD__) && !defined(ARCADIA_BUILD) @@ -369,7 +369,7 @@ static void toStringEveryLineImpl(const void * const * frame_pointers, size_t of #endif } -static std::string toStringImpl(const void * const * frame_pointers, size_t offset, size_t size) +static std::string toStringImpl(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size) { std::stringstream out; toStringEveryLineImpl(frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); @@ -378,7 +378,7 @@ static std::string toStringImpl(const void * const * frame_pointers, size_t offs void StackTrace::toStringEveryLine(std::function callback) const { - toStringEveryLineImpl(frame_pointers.data(), offset, size, std::move(callback)); + toStringEveryLineImpl(frame_pointers, offset, size, std::move(callback)); } @@ -388,7 +388,7 @@ std::string StackTrace::toString() const /// We use simple cache because otherwise the server could be overloaded by trash queries. static SimpleCache func_cached; - return func_cached(frame_pointers.data(), offset, size); + return func_cached(frame_pointers, offset, size); } std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t size) @@ -400,5 +400,5 @@ std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t frame_pointers_copy[i] = frame_pointers_[i]; static SimpleCache func_cached; - return func_cached(frame_pointers_copy.data(), offset, size); + return func_cached(frame_pointers_copy, offset, size); } From 3ed76a388e20636441472331c1a7a73f43f9b5fe Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 00:27:04 +0300 Subject: [PATCH 77/82] send build_id to sentry as well + fix some minor issues --- base/daemon/BaseDaemon.cpp | 9 ++++++++- base/daemon/SentryWriter.cpp | 6 +++++- base/daemon/SentryWriter.h | 3 ++- src/Common/StackTrace.cpp | 2 +- src/Common/StackTrace.h | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 204f16078cb..e7ccf84d7da 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -298,7 +298,14 @@ private: stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); /// Send crash report to developers (if configured) - SentryWriter::onFault(sig, info, context, stack_trace); + + #if defined(__ELF__) && !defined(__FreeBSD__) + const String & build_id_hex = DB::SymbolIndex::instance().getBuildIDHex(); + #else + String build_id_hex{}; + #endif + + SentryWriter::onFault(sig, info, context, stack_trace, build_id_hex); /// When everything is done, we will try to send these error messages to client. if (thread_ptr) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index ab560f4d8a1..298e069092a 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -161,7 +161,7 @@ void SentryWriter::shutdown() #endif } -void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace) +void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace, const String & build_id_hex) { #if USE_SENTRY auto * logger = &Poco::Logger::get("SentryWriter"); @@ -171,6 +171,10 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str()); sentry_set_tag("signal", strsignal(sig)); sentry_set_extra("signal_number", sentry_value_new_int32(sig)); + if (!build_id_hex.empty()) + { + sentry_set_tag("build_id", build_id_hex.c_str()); + } setExtras(); /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/stacktrace/ diff --git a/base/daemon/SentryWriter.h b/base/daemon/SentryWriter.h index 655a4e93bfd..a7b255e72bf 100644 --- a/base/daemon/SentryWriter.h +++ b/base/daemon/SentryWriter.h @@ -27,6 +27,7 @@ public: int sig, const siginfo_t & info, const ucontext_t & context, - const StackTrace & stack_trace + const StackTrace & stack_trace, + const String & build_id_hex ); }; diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 1ea5d100440..9c6904b884c 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -308,7 +308,7 @@ const StackTrace::FramePointers & StackTrace::getFramePointers() const return frame_pointers; } -static void toStringEveryLineImpl(const void * const * frame_pointers, size_t offset, size_t size, std::function callback) +static void toStringEveryLineImpl(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size, std::function callback) { if (size == 0) return callback(""); diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 374f0314533..3ae4b964838 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -54,7 +54,7 @@ public: std::string toString() const; static std::string toString(void ** frame_pointers, size_t offset, size_t size); - static void symbolize(const void * const * frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); + static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); void toStringEveryLine(std::function callback) const; protected: From db568b477eca56b2953658ef55e0edc593e7c7a2 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 11:12:47 +0300 Subject: [PATCH 78/82] lost unused --- base/daemon/SentryWriter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 298e069092a..ea93d09f9aa 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -245,5 +245,6 @@ void SentryWriter::onFault(int sig, const siginfo_t & info, const ucontext_t & c UNUSED(info); UNUSED(context); UNUSED(stack_trace); + UNUSED(build_id_hex); #endif } From ea2d1856f3830e65641c5d9239773e9a11c0d897 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 15:35:22 +0300 Subject: [PATCH 79/82] try to rename py files to less common names --- .../{http_server.py => fake_sentry_server.py} | 0 .../{test.py => test_send_crash_reports.py} | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename tests/integration/test_send_crash_reports/{http_server.py => fake_sentry_server.py} (100%) rename tests/integration/test_send_crash_reports/{test.py => test_send_crash_reports.py} (77%) diff --git a/tests/integration/test_send_crash_reports/http_server.py b/tests/integration/test_send_crash_reports/fake_sentry_server.py similarity index 100% rename from tests/integration/test_send_crash_reports/http_server.py rename to tests/integration/test_send_crash_reports/fake_sentry_server.py diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test_send_crash_reports.py similarity index 77% rename from tests/integration/test_send_crash_reports/test.py rename to tests/integration/test_send_crash_reports/test_send_crash_reports.py index f9e95f953d0..ff4b55da99b 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test_send_crash_reports.py @@ -5,7 +5,7 @@ import pytest import helpers.cluster import helpers.test_tools -import http_server +import fake_sentry_server SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -25,15 +25,15 @@ def started_node(): def test_send_segfault(started_node,): - started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "http_server.py"), "/http_server.py") - started_node.exec_in_container(["bash", "-c", "python2 /http_server.py"], detach=True, user="root") + started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") + started_node.exec_in_container(["bash", "-c", "python2 /fake_sentry_server.py"], detach=True, user="root") time.sleep(0.5) started_node.exec_in_container(["bash", "-c", "pkill -11 clickhouse"], user="root") result = None for attempt in range(1, 6): time.sleep(0.25 * attempt) - result = started_node.exec_in_container(['cat', http_server.RESULT_PATH], user='root') + result = started_node.exec_in_container(['cat', fake_sentry_server.RESULT_PATH], user='root') if result == 'OK': break elif result == 'INITIAL_STATE': From 3206ed832a6511a002403cdfb691aa905914ce9f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 15:54:04 +0300 Subject: [PATCH 80/82] add __init__.py --- tests/integration/test_send_crash_reports/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_send_crash_reports/__init__.py diff --git a/tests/integration/test_send_crash_reports/__init__.py b/tests/integration/test_send_crash_reports/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From a2142ae7d1e9d8f809fbf0a92a3c777e3af43f30 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 24 Jun 2020 22:29:11 +0300 Subject: [PATCH 81/82] trigger ci From 269fc138be07ac737309f0b403427577dd07ab6a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 25 Jun 2020 11:11:29 +0300 Subject: [PATCH 82/82] rename test back to test.py --- .../{test_send_crash_reports.py => test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/integration/test_send_crash_reports/{test_send_crash_reports.py => test.py} (100%) diff --git a/tests/integration/test_send_crash_reports/test_send_crash_reports.py b/tests/integration/test_send_crash_reports/test.py similarity index 100% rename from tests/integration/test_send_crash_reports/test_send_crash_reports.py rename to tests/integration/test_send_crash_reports/test.py