From 48fe30e5a28592f326a38d8d8261e782a7b0ba7d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 25 Mar 2021 23:41:03 +0300 Subject: [PATCH 1/3] Add missing logging for exception in InterserverIOHTTPHandler --- src/Server/InterserverIOHTTPHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 740072e8e9f..7d8dfaaf2c8 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -107,6 +107,7 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe } catch (...) { + tryLogCurrentException(log); out.finalize(); } }; From e1de9600253c741e0d8f4b15c5c64dab658544b0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 25 Mar 2021 23:41:03 +0300 Subject: [PATCH 2/3] Fix uncaught exception in InterserverIOHTTPHandler There was one more uncaught exception case [1]: 2021.03.19 18:11:00.845632 [ 17469 ] {} InterserverIOHTTPHandler: Done processing query ... 2021.03.19 18:11:31.698961 [ 80145 ] {} BaseDaemon: ######################################## 2021.03.19 18:11:31.699903 [ 80145 ] {} BaseDaemon: (version 21.4.1.6293 (official build), build id: 859E400E1C65C4702FE491420741DD8B58190002) (from thread 17469) (no query) Received signal Aborted (6) 2021.03.19 18:11:32.614075 [ 80145 ] {} BaseDaemon: 8. ./obj-x86_64-linux-gnu/../contrib/libcxxabi/src/cxa_handlers.cpp:89: std::terminate() @ 0x21e9b3a2 in /usr/bin/clickhouse 2021.03.19 18:11:43.831215 [ 80145 ] {} BaseDaemon: 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp:201: ? @ 0x1be77038 in /usr/bin/clickhouse 2021.03.19 18:11:44.743193 [ 80145 ] {} BaseDaemon: 11. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:892: std::__1::allocator::destroy(DB::WriteBufferFromHTTPServerResponse*) @ 0x1bddd7c9 in /usr/bin/clickhouse 2021.03.19 18:11:45.283905 [ 80145 ] {} BaseDaemon: 12. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:541: void std::__1::allocator_traits >::__destroy(std::__1::integral_constant, std::__1::allocator&, DB::WriteBufferFromHTTPServerResponse*) @ 0x1bddd79d in /usr/bin/clickhouse 2021.03.19 18:11:45.805233 [ 80145 ] {} BaseDaemon: 13. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:487: void std::__1::allocator_traits >::destroy(std::__1::allocator&, DB::WriteBufferFromHTTPServerResponse*) @ 0x1bddd76d in /usr/bin/clickhouse 2021.03.19 18:11:46.351371 [ 80145 ] {} BaseDaemon: 14. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2611: std::__1::__shared_ptr_emplace >::__on_zero_shared() @ 0x1bddd525 in /usr/bin/clickhouse 2021.03.19 18:11:46.579263 [ 80145 ] {} BaseDaemon: 15. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2476: std::__1::__shared_count::__release_shared() @ 0x119490ed in /usr/bin/clickhouse 2021.03.19 18:11:46.790912 [ 80145 ] {} BaseDaemon: 16. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2517: std::__1::__shared_weak_count::__release_shared() @ 0x1194908f in /usr/bin/clickhouse 2021.03.19 18:11:47.277990 [ 80145 ] {} BaseDaemon: 17. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3213: std::__1::shared_ptr::~shared_ptr() @ 0x1bdd75fc in /usr/bin/clickhouse 2021.03.19 18:11:47.649213 [ 80145 ] {} BaseDaemon: 18. ./obj-x86_64-linux-gnu/../src/Server/InterserverIOHTTPHandler.h:34: DB::InterserverIOHTTPHandler::Output::~Output() @ 0x1bdf6bd5 in /usr/bin/clickhouse 2021.03.19 18:11:47.921556 [ 80145 ] {} BaseDaemon: 19. ./obj-x86_64-linux-gnu/../src/Server/InterserverIOHTTPHandler.cpp:154: DB::InterserverIOHTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x1bdf653f in /usr/bin/clickhouse [1]: https://clickhouse-test-reports.s3.yandex.net/0/78c56b891383288cf3a893139e796fc87476412e/stress_test_(debug).html Since in case of no errors during processing we should call finalize, to ensure that it will not be called from dtor. Fixes: #22046 Fixes: #22067 --- src/Server/InterserverIOHTTPHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 7d8dfaaf2c8..426e4ca2138 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -117,6 +117,7 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe if (auto [message, success] = checkAuthentication(request); success) { processQuery(request, response, used_output); + used_output.out->finalize(); LOG_DEBUG(log, "Done processing query"); } else From 50003e496a1551a49733b3c67b1b0fb4939af54b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 25 Mar 2021 23:41:03 +0300 Subject: [PATCH 3/3] Use existing logger for logging from WriteBufferFromS3 dtor --- src/IO/WriteBufferFromS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 7373b24991a..93aaf9456b5 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -120,7 +120,7 @@ WriteBufferFromS3::~WriteBufferFromS3() } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + tryLogCurrentException(log); } }