From 8cb0bb4ca82ac57ee6b886b7a8b63c46968077e0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 20 Feb 2021 16:46:55 +0300 Subject: [PATCH 1/6] Fix SIGSEGV by waiting servers thread pool It is easy to reproduce with shutdown_wait_unfinished=0: ================================================================= ==13442==ERROR: AddressSanitizer: heap-use-after-free on address 0x611000210f30 at pc 0x00000a8e55a0 bp 0x7fff2b83e270 sp 0x7fff2b83e268 WRITE of size 8 at 0x611000210f30 thread T2 (TCPHandler) 0 0xa8e559f in long std::__1::__cxx_atomic_fetch_add(std::__1::__cxx_atomic_base_impl*, long, std::__1::memory_order) obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:1050:12 1 0xa8e559f in std::__1::__atomic_base::fetch_add(long, std::__1::memory_order) obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:1719:17 2 0xa8e559f in MemoryTracker::alloc(long) obj-x86_64-linux-gnu/../src/Common/MemoryTracker.cpp:146:35 3 0xa8e510c in MemoryTracker::alloc(long) obj-x86_64-linux-gnu/../src/Common/MemoryTracker.cpp 4 0xa90b474 in DB::ThreadStatus::~ThreadStatus() obj-x86_64-linux-gnu/../src/Common/ThreadStatus.cpp:92:28 5 0x1f90ee83 in DB::TCPHandler::runImpl() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:450:1 6 0x1f92dcac in DB::TCPHandler::run() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:1492:9 7 0x25bdc2fe in Poco::Net::TCPServerConnection::start() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:43:3 8 0x25bdce1b in Poco::Net::TCPServerDispatcher::run() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:113:19 9 0x25e9c784 in Poco::PooledThread::run() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:199:14 10 0x25e96cd6 in Poco::ThreadImpl::runnableEntry(void*) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:345:27 11 0x7ffff7f723e8 in start_thread (/usr/lib/libpthread.so.0+0x93e8) 12 0x7ffff7ea0292 in clone (/usr/lib/libc.so.6+0x100292) 0x611000210f30 is located 112 bytes inside of 216-byte region [0x611000210ec0,0x611000210f98) freed by thread T0 here: 0 0xa845d02 in operator delete(void*, unsigned long) (/src/ch/tmp/upstream/clickhouse-asan+0xa845d02) 1 0x1d38328c in void std::__1::__libcpp_operator_delete(void*, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/new:245:3 2 0x1d38328c in void std::__1::__do_deallocate_handle_size<>(void*, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/new:271:10 3 0x1d38328c in std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/new:285:14 4 0x1d38328c in std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser>, void*> >::deallocate(std::__1::__hash_node, std::__1::allocator >, DB::ProcessListForUser>, void*>*, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:849:13 5 0x1d38328c in std::__1::allocator_traits, std::__1::allocator >, DB::ProcessListForUser>, void*> > >::deallocate(std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser>, void*> >&, std::__1::__hash_node, std::__1::allocator >, DB::ProcessListForUser>, void*>*, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:476:14 6 0x1d38328c in std::__1::__hash_table, std::__1::allocator >, DB::ProcessListForUser>, std::__1::__unordered_map_hasher, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, true>, std::__1::__unordered_map_equal, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::equal_to, std::__1::allocator > >, std::__1::hash, std::__1::allocator > >, true>, std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser> > >::__deallocate_node(std::__1::__hash_node_base, std::__1::allocator >, DB::ProcessListForUser>, void*>*>*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__hash_table:1581:9 7 0x1d38328c in std::__1::__hash_table, std::__1::allocator >, DB::ProcessListForUser>, std::__1::__unordered_map_hasher, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, true>, std::__1::__unordered_map_equal, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::equal_to, std::__1::allocator > >, std::__1::hash, std::__1::allocator > >, true>, std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser> > >::~__hash_table() obj-x86_64-linux-gnu/../contrib/libcxx/include/__hash_table:1519:5 8 0x1d38328c in std::__1::unordered_map, std::__1::allocator >, DB::ProcessListForUser, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, std::__1::allocator, std::__1::allocator > const, DB::ProcessListForUser> > >::~unordered_map() obj-x86_64-linux-gnu/../contrib/libcxx/include/unordered_map:1044:5 9 0x1d38328c in DB::ProcessList::~ProcessList() obj-x86_64-linux-gnu/../src/Interpreters/ProcessList.h:263:7 10 0x1d38169c in DB::ContextShared::~ContextShared() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:417:5 11 0x1d32f3e5 in std::__1::default_delete::operator()(DB::ContextShared*) const obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:1397:5 12 0x1d32f3e5 in std::__1::unique_ptr >::reset(DB::ContextShared*) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:1658:7 13 0x1d32f3e5 in DB::SharedContextHolder::reset() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:485:44 14 0xa8863d4 in DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:880:5 15 0xa8863d4 in ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::invoke() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:97:9 16 0xa8863d4 in ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::~basic_scope_guard() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:47:28 17 0xa86d889 in DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:1379:1 18 0x25c0c8b5 in Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 19 0xa85070d in DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:340:25 20 0x25c49eb7 in Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 21 0xa84cd11 in mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:132:20 22 0xa848c3a in main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 23 0x7ffff7dc8151 in __libc_start_main (/usr/lib/libc.so.6+0x28151) previously allocated by thread T2 (TCPHandler) here: 0 0xa84509d in operator new(unsigned long) (/src/ch/tmp/upstream/clickhouse-asan+0xa84509d) 1 0x1e2a7aa6 in void* std::__1::__libcpp_operator_new(unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/new:235:10 2 0x1e2a7aa6 in std::__1::__libcpp_allocate(unsigned long, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/new:261:10 3 0x1e2a7aa6 in std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser>, void*> >::allocate(unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:840:38 4 0x1e2a7aa6 in std::__1::allocator_traits, std::__1::allocator >, DB::ProcessListForUser>, void*> > >::allocate(std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser>, void*> >&, unsigned long) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:468:21 5 0x1e2a7aa6 in std::__1::unique_ptr, std::__1::allocator >, DB::ProcessListForUser>, void*>, std::__1::__hash_node_destructor, std::__1::allocator >, DB::ProcessListForUser>, void*> > > > std::__1::__hash_table, std::__1::allocator >, DB::ProcessListForUser>, std::__1::__unordered_map_hasher, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, true>, std::__1::__unordered_map_equal, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::equal_to, std::__1::allocator > >, std::__1::hash, std::__1::allocator > >, true>, std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser> > >::__construct_node_hash, std::__1::allocator > const&>, std::__1::tuple<> >(unsigned long, std::__1::piecewise_construct_t const&, std::__1::tuple, std::__1::allocator > const&>&&, std::__1::tuple<>&&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__hash_table:2472:23 6 0x1e2a7aa6 in std::__1::pair, std::__1::allocator >, DB::ProcessListForUser>, void*>*>, bool> std::__1::__hash_table, std::__1::allocator >, DB::ProcessListForUser>, std::__1::__unordered_map_hasher, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, true>, std::__1::__unordered_map_equal, std::__1::allocator >, std::__1::__hash_value_type, std::__1::allocator >, DB::ProcessListForUser>, std::__1::equal_to, std::__1::allocator > >, std::__1::hash, std::__1::allocator > >, true>, std::__1::allocator, std::__1::allocator >, DB::ProcessListForUser> > >::__emplace_unique_key_args, std::__1::allocator >, std::__1::piecewise_construct_t const&, std::__1::tuple, std::__1::allocator > const&>, std::__1::tuple<> >(std::__1::basic_string, std::__1::allocator > const&, std::__1::piecewise_construct_t const&, std::__1::tuple, std::__1::allocator > const&>&&, std::__1::tuple<>&&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__hash_table:2093:29 7 0x1e29c13d in std::__1::unordered_map, std::__1::allocator >, DB::ProcessListForUser, std::__1::hash, std::__1::allocator > >, std::__1::equal_to, std::__1::allocator > >, std::__1::allocator, std::__1::allocator > const, DB::ProcessListForUser> > >::operator[](std::__1::basic_string, std::__1::allocator > const&) obj-x86_64-linux-gnu/../contrib/libcxx/include/unordered_map:1740:21 8 0x1e29c13d in DB::ProcessList::insert(std::__1::basic_string, std::__1::allocator > const&, DB::IAST const*, DB::Context&) obj-x86_64-linux-gnu/../src/Interpreters/ProcessList.cpp:183:50 9 0x1e5a3a58 in DB::executeQueryImpl(char const*, char const*, DB::Context&, bool, DB::QueryProcessingStage::Enum, bool, DB::ReadBuffer*) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:486:59 10 0x1e5a153e in DB::executeQuery(std::__1::basic_string, std::__1::allocator > const&, DB::Context&, bool, DB::QueryProcessingStage::Enum, bool) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:904:30 11 0x1f909bdc in DB::TCPHandler::runImpl() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:289:24 12 0x1f92dcac in DB::TCPHandler::run() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:1492:9 13 0x25bdc2fe in Poco::Net::TCPServerConnection::start() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:43:3 14 0x25bdce1b in Poco::Net::TCPServerDispatcher::run() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:113:19 15 0x25e9c784 in Poco::PooledThread::run() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:199:14 16 0x25e96cd6 in Poco::ThreadImpl::runnableEntry(void*) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:345:27 17 0x7ffff7f723e8 in start_thread (/usr/lib/libpthread.so.0+0x93e8) Thread T2 (TCPHandler) created by T0 here: 0 0xa7ffe0a in pthread_create (/src/ch/tmp/upstream/clickhouse-asan+0xa7ffe0a) 1 0x25e9606f in Poco::ThreadImpl::startImpl(Poco::SharedPtr >) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:202:6 2 0x25e98eea in Poco::Thread::start(Poco::Runnable&) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:128:2 3 0x25e9cd28 in Poco::PooledThread::start() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:85:10 4 0x25e9cd28 in Poco::ThreadPool::ThreadPool(int, int, int, int) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:252:12 5 0xa865aff in DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:831:22 6 0x25c0c8b5 in Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 7 0xa85070d in DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:340:25 8 0x25c49eb7 in Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 9 0xa84cd11 in mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:132:20 10 0xa848c3a in main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 11 0x7ffff7dc8151 in __libc_start_main (/usr/lib/libc.so.6+0x28151) SUMMARY: AddressSanitizer: heap-use-after-free obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:1050:12 in long std::__1::__cxx_atomic_fetch_add(std::__1::__cxx_atomic_base_impl*, long, std::__1::memory_order) Shadow bytes around the buggy address: 0x0c228003a190: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c228003a1a0: fd fd fd fd fd fd fa fa fa fa fa fa fa fa fa fa 0x0c228003a1b0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c228003a1c0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fa fa 0x0c228003a1d0: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd =>0x0c228003a1e0: fd fd fd fd fd fd[fd]fd fd fd fd fd fd fd fd fd 0x0c228003a1f0: fd fd fd fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c228003a200: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c228003a210: fd fd fd fd fd fd fd fd fd fa fa fa fa fa fa fa 0x0c228003a220: fa fa fa fa fa fa fa fa fd fd fd fd fd fd fd fd 0x0c228003a230: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Freed heap region: fd Stack left redzone: f1 Stack mid redzone: f2 Stack right redzone: f3 Stack after return: f5 Stack use after scope: f8 Global redzone: f9 Global init order: f6 Poisoned by user: f7 Container overflow: fc Array cookie: ac Intra object redzone: bb ASan internal: fe Left alloca redzone: ca Right alloca redzone: cb Shadow gap: cc ==13442==ABORTING 2021.02.20 16:39:50.861426 [ 13443 ] {} BaseDaemon: Received signal -3 2021.02.20 16:39:50.861668 [ 14989 ] {} BaseDaemon: ######################################## 2021.02.20 16:39:50.861749 [ 14989 ] {} BaseDaemon: (version 21.3.1.6073 (official build), build id: AC8A516D2F60B8505FA128074527EC2C86198E64) (from thread 13874) (no query) Received signal Unknown signal (-3) 2021.02.20 16:39:50.861810 [ 14989 ] {} BaseDaemon: Sanitizer trap. 2021.02.20 16:39:50.861880 [ 14989 ] {} BaseDaemon: Stack trace: 0xa8e94a7 0xad25b1b 0xa831a16 0xa819444 0xa81aefe 0xa81bb4b 0xa8e55a0 0xa8e510d 0xa90b475 0x1f90ee84 0x1f92dcad 0x25bdc2ff 0x25bdce1c 0x25e9c785 0x25e96cd7 0x7ffff7f723e9 0x7ffff7ea0293 2021.02.20 16:39:50.903643 [ 14989 ] {} BaseDaemon: 0.1. inlined from ./obj-x86_64-linux-gnu/../src/Common/StackTrace.cpp:298: StackTrace::tryCapture() 2021.02.20 16:39:50.903708 [ 14989 ] {} BaseDaemon: 0. ../src/Common/StackTrace.cpp:259: StackTrace::StackTrace() @ 0xa8e94a7 in /src/ch/tmp/upstream/clickhouse-asan 2021.02.20 16:39:51.041733 [ 14989 ] {} BaseDaemon: 1.1. inlined from ./obj-x86_64-linux-gnu/../src/Common/CurrentThread.h:78: DB::CurrentThread::getQueryId() 2021.02.20 16:39:51.041768 [ 14989 ] {} BaseDaemon: 1. ../base/daemon/BaseDaemon.cpp:381: sanitizerDeathCallback() @ 0xad25b1b in /src/ch/tmp/upstream/clickhouse-asan 2021.02.20 16:39:52.551623 [ 13442 ] {} Application: shutting down 2021.02.20 16:39:52.551696 [ 13442 ] {} Application: Uninitializing subsystem: Logging Subsystem 2021.02.20 16:39:52.551792 [ 13443 ] {} BaseDaemon: Received signal -2 2021.02.20 16:39:52.551831 [ 13443 ] {} BaseDaemon: Stop SignalListener thread --- programs/server/Server.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8a96612721d..cb53687df4d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -951,6 +951,9 @@ int Server::main(const std::vector & /*args*/) global_context->shutdownKeeperStorageDispatcher(); } + /// Wait server pool to avoid use-after-free of destroyed context in the handlers + server_pool.joinAll(); + /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. */ From 3056d2c5d1bfb5ad4c8627d45307c01f76d6eaca Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 1 Mar 2021 08:10:13 +0300 Subject: [PATCH 2/6] Fix data-race during server shutdown the context Found with 01737_clickhouse_server_wait_server_pool: ================== WARNING: ThreadSanitizer: data race (pid=13248) Write of size 1 at 0x7b9000003a38 by main thread: 0 std::__1::__optional_destruct_base::reset() obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:246:24 (clickhouse-tsan+0x11e3043e) 1 DB::ContextShared::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:441:21 (clickhouse-tsan+0x11e3043e) 2 DB::Context::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:2249:13 (clickhouse-tsan+0x11e28c17) 3 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:892:5 (clickhouse-tsan+0x8ab1a32) 4 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::invoke() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:97:9 (clickhouse-tsan+0x8ab1a32) 5 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::~basic_scope_guard() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:47:28 (clickhouse-tsan+0x8ab1a32) 6 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:1395:1 (clickhouse-tsan+0x8aacb39) 7 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b3446b) 8 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa04be) 9 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b50883) 10 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8a9f08e) 11 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8a9d5f9) Previous read of size 1 at 0x7b9000003a38 by thread T2 (mutexes: write M2504): 0 std::__1::__optional_storage_base::has_value() const obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:295:22 (clickhouse-tsan+0x11e25348) 1 std::__1::optional::operator bool() const obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:938:64 (clickhouse-tsan+0x11e25348) 2 DB::Context::getQueryLog() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1875:10 (clickhouse-tsan+0x11e25348) 3 DB::executeQueryImpl(char const*, char const*, DB::Context&, bool, DB::QueryProcessingStage::Enum, bool, DB::ReadBuffer*) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:657:50 (clickhouse-tsan+0x12653751) 4 DB::executeQuery(std::__1::basic_string, std::__1::allocator > const&, DB::Context&, bool, DB::QueryProcessingStage::Enum, bool) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:904:30 (clickhouse-tsan+0x12651308) 5 DB::TCPHandler::runImpl() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:289:24 (clickhouse-tsan+0x12f04b45) 6 DB::TCPHandler::run() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:1500:9 (clickhouse-tsan+0x12f13907) 7 Poco::Net::TCPServerConnection::start() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:43:3 (clickhouse-tsan+0x15b1f722) 8 Poco::Net::TCPServerDispatcher::run() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:113:19 (clickhouse-tsan+0x15b1fe4e) 9 Poco::PooledThread::run() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:199:14 (clickhouse-tsan+0x15c86fe1) 10 Poco::(anonymous namespace)::RunnableHolder::run() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:55:11 (clickhouse-tsan+0x15c8557f) 11 Poco::ThreadImpl::runnableEntry(void*) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:345:27 (clickhouse-tsan+0x15c83d87) Location is heap block of size 7296 at 0x7b9000002000 allocated by main thread: 0 operator new(unsigned long) (clickhouse-tsan+0x8a9aae7) 1 std::__1::__unique_if::__unique_single std::__1::make_unique() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2068:28 (clickhouse-tsan+0x11e15c2c) 2 DB::Context::createShared() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:503:32 (clickhouse-tsan+0x11e15c2c) 3 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:426:27 (clickhouse-tsan+0x8aa19ee) 4 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b3446b) 5 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa04be) 6 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b50883) 7 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8a9f08e) 8 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8a9d5f9) Mutex M2504 (0x7b9000002008) created at: 0 pthread_mutex_init (clickhouse-tsan+0x8a0d37d) 1 std::__1::__libcpp_recursive_mutex_init(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:370:10 (clickhouse-tsan+0x17cc4d93) 2 std::__1::recursive_mutex::recursive_mutex() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:56:14 (clickhouse-tsan+0x17cc4d93) 3 DB::ContextShared::ContextShared() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:394:5 (clickhouse-tsan+0x11e40bc3) 4 std::__1::__unique_if::__unique_single std::__1::make_unique() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2068:32 (clickhouse-tsan+0x11e15c37) 5 DB::Context::createShared() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:503:32 (clickhouse-tsan+0x11e15c37) 6 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:426:27 (clickhouse-tsan+0x8aa19ee) 7 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b3446b) 8 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa04be) 9 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b50883) 10 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8a9f08e) 11 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8a9d5f9) Thread T2 'TCPHandler' (tid=13643, running) created by main thread at: 0 pthread_create (clickhouse-tsan+0x8a0bf0b) 1 Poco::ThreadImpl::startImpl(Poco::SharedPtr >) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:202:6 (clickhouse-tsan+0x15c83827) 2 Poco::Thread::start(Poco::Runnable&) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:128:2 (clickhouse-tsan+0x15c84f6c) 3 Poco::PooledThread::start() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:85:10 (clickhouse-tsan+0x15c873e2) 4 Poco::ThreadPool::ThreadPool(int, int, int, int) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:252:12 (clickhouse-tsan+0x15c873e2) 5 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:843:22 (clickhouse-tsan+0x8aa8e5f) 6 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b3446b) 7 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa04be) 8 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b50883) 9 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8a9f08e) 10 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8a9d5f9) SUMMARY: ThreadSanitizer: data race obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:246:24 in std::__1::__optional_destruct_base::reset() v2: fix deadlock by calling SystemLogs::shutdown w/o Context lock --- src/Interpreters/Context.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 187edf8843f..ab327089333 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -442,6 +442,8 @@ struct ContextSharedPart DatabaseCatalog::shutdown(); + auto lock = std::lock_guard(mutex); + /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. From a61ae26729c34e31e4ad02c96105e81a1a5db6a7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 2 Mar 2021 23:41:29 +0300 Subject: [PATCH 3/6] Fix lock-order-inversion during system.*_log shutting down As TSan reports [1]: WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=24429) Cycle in lock order graph: M152695175523663992 (0x000000000000) => M2505 (0x7b9000002008) => M152695175523663992 Mutex M2505 acquired here while holding mutex M152695175523663992 in thread T7: 0 pthread_mutex_lock (clickhouse-tsan+0x8a301b6) 1 std::__1::__libcpp_recursive_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:385:10 (clickhouse-tsan+0x17cd6e89) 2 std::__1::recursive_mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:71:14 (clickhouse-tsan+0x17cd6e89) 3 std::__1::unique_lock::unique_lock(std::__1::recursive_mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:119:61 (clickhouse-tsan+0x11e32a9f) 4 DB::Context::getLock() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:517:12 (clickhouse-tsan+0x11e32a9f) 5 DB::Context::getSchedulePool() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1517:17 (clickhouse-tsan+0x11e32a9f) 6 DB::IBackgroundJobExecutor::start() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:158:42 (clickhouse-tsan+0x12bde50a) 7 DB::StorageMergeTree::startup() obj-x86_64-linux-gnu/../src/Storages/StorageMergeTree.cpp:112:29 (clickhouse-tsan+0x129e9e1e) 8 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_2::operator()(std::__1::shared_ptr const&) const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:230:16 (clickhouse-tsan+0x11d6fa2a) 9 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3::operator()() const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:238:56 (clickhouse-tsan+0x11d6fa2a) 10 decltype(std::__1::forward&)::$_3&>(fp)()) std::__1::__invoke&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x11d6fa2a) 11 void std::__1::__invoke_void_return_wrapper::__call&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x11d6fa2a) 12 std::__1::__function::__default_alloc_func&)::$_3, void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x11d6fa2a) 13 void std::__1::__function::__policy_invoker::__call_impl&)::$_3, void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x11d6fa2a) 14 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b39350) 15 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b39350) 16 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b39350) 17 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3c070) 18 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(fp0)...)) std::__1::__invoke_constexpr::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3682:1 (clickhouse-tsan+0x8b3c070) 19 decltype(auto) std::__1::__apply_tuple_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::__tuple_indices&...>) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1415:1 (clickhouse-tsan+0x8b3c070) 20 decltype(auto) std::__1::apply::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1424:1 (clickhouse-tsan+0x8b3c070) 21 ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()::operator()() obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:178:13 (clickhouse-tsan+0x8b3c070) 22 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3bfd1) 23 void std::__1::__invoke_void_return_wrapper::__call::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x8b3bfd1) 24 std::__1::__function::__default_alloc_func::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x8b3bfd1) 25 void std::__1::__function::__policy_invoker::__call_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x8b3bfd1) 26 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b36c75) 27 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b36c75) 28 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b36c75) 29 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3a918) 30 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3a918) 31 void std::__1::__thread_execute >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(std::__1::tuple::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>&, std::__1::__tuple_indices<>) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:280:5 (clickhouse-tsan+0x8b3a918) 32 void* std::__1::__thread_proxy >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()> >(void*) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:291:5 (clickhouse-tsan+0x8b3a918) Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative warning message Mutex M152695175523663992 acquired here while holding mutex M2505 in main thread: 0 pthread_mutex_lock (clickhouse-tsan+0x8a301b6) 1 std::__1::__libcpp_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:405:10 (clickhouse-tsan+0x17cd6cf9) 2 std::__1::mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:33:14 (clickhouse-tsan+0x17cd6cf9) 3 std::__1::lock_guard::lock_guard(std::__1::mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse-tsan+0x12bdee4b) 4 DB::IBackgroundJobExecutor::finish() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:167:21 (clickhouse-tsan+0x12bdee4b) 5 DB::IBackgroundJobExecutor::~IBackgroundJobExecutor() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:185:5 (clickhouse-tsan+0x12bdee4b) 6 DB::StorageMergeTree::~StorageMergeTree() obj-x86_64-linux-gnu/../src/Storages/StorageMergeTree.cpp:174:1 (clickhouse-tsan+0x129ea118) 7 std::__1::default_delete::operator()(DB::StorageMergeTree*) const obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:1397:5 (clickhouse-tsan+0x12e4433b) 8 std::__1::__shared_ptr_pointer::__shared_ptr_default_delete, std::__1::allocator >::__on_zero_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2565:5 (clickhouse-tsan+0x12e4433b) 9 std::__1::__shared_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2475:9 (clickhouse-tsan+0x125b355a) 10 std::__1::__shared_weak_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2517:27 (clickhouse-tsan+0x125b355a) 11 std::__1::shared_ptr::~shared_ptr() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3212:19 (clickhouse-tsan+0x125b355a) 12 DB::SystemLog::~SystemLog() obj-x86_64-linux-gnu/../src/Interpreters/SystemLog.h:118:7 (clickhouse-tsan+0x125b355a) 13 std::__1::allocator::destroy(DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:891:15 (clickhouse-tsan+0x125aff68) 14 void std::__1::allocator_traits >::__destroy(std::__1::integral_constant, std::__1::allocator&, DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:539:21 (clickhouse-tsan+0x125aff68) 15 void std::__1::allocator_traits >::destroy(std::__1::allocator&, DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:487:14 (clickhouse-tsan+0x125aff68) 16 std::__1::__shared_ptr_emplace >::__on_zero_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2611:9 (clickhouse-tsan+0x125aff68) 17 std::__1::__shared_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2475:9 (clickhouse-tsan+0x1258e74f) 18 std::__1::__shared_weak_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2517:27 (clickhouse-tsan+0x1258e74f) 19 std::__1::shared_ptr::~shared_ptr() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3212:19 (clickhouse-tsan+0x1258e74f) 20 DB::SystemLogs::~SystemLogs() obj-x86_64-linux-gnu/../src/Interpreters/SystemLog.cpp:155:1 (clickhouse-tsan+0x1258e74f) 21 std::__1::__optional_destruct_base::reset() obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:245:21 (clickhouse-tsan+0x11e41085) 22 DB::ContextShared::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:443:21 (clickhouse-tsan+0x11e41085) 23 DB::Context::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:2251:13 (clickhouse-tsan+0x11e39867) 24 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:892:5 (clickhouse-tsan+0x8ab8732) 25 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::invoke() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:97:9 (clickhouse-tsan+0x8ab8732) 26 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::~basic_scope_guard() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:47:28 (clickhouse-tsan+0x8ab8732) 27 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:1395:1 (clickhouse-tsan+0x8ab3839) 28 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b464ab) 29 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa71be) 30 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b628c3) 31 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8aa5d8e) 32 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8aa42f9) [1]: https://clickhouse-test-reports.s3.yandex.net/21318/38be9ff43ac4c46ce6e803fc125d910bde1d4c71/functional_stateful_tests_(thread).html#fail1 --- src/Interpreters/SystemLog.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 2f7bdd4a22f..aa01ca3517b 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -152,6 +152,8 @@ public: void shutdown() override { stopFlushThread(); + if (table) + table->shutdown(); } String getName() override From d59bdfd45d24a9400b02571712318a2ee3ce47ba Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 3 Mar 2021 08:08:10 +0300 Subject: [PATCH 4/6] Fix one more lock-order-inversion TSan report [1]: WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=11314) Cycle in lock order graph: M183938897938677368 (0x000000000000) => M2505 (0x7b9000002008) => M183938897938677368 Mutex M2505 acquired here while holding mutex M183938897938677368 in thread T6: 0 pthread_mutex_lock (clickhouse-tsan+0x8a327b6) 1 std::__1::__libcpp_recursive_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:385:10 (clickhouse-tsan+0x17cdb689) 2 std::__1::recursive_mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:71:14 (clickhouse-tsan+0x17cdb689) 3 std::__1::unique_lock::unique_lock(std::__1::recursive_mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:119:61 (clickhouse-tsan+0x11e3506f) 4 DB::Context::getLock() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:517:12 (clickhouse-tsan+0x11e3506f) 5 DB::Context::getSchedulePool() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1517:17 (clickhouse-tsan+0x11e3506f) 6 DB::IBackgroundJobExecutor::start() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:158:42 (clickhouse-tsan+0x12be1cda) 7 DB::StorageMergeTree::startup() obj-x86_64-linux-gnu/../src/Storages/StorageMergeTree.cpp:112:29 (clickhouse-tsan+0x129ed46e) 8 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_2::operator()(std::__1::shared_ptr const&) const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:230:16 (clickhouse-tsan+0x11d71fba) 9 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3::operator()() const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:238:56 (clickhouse-tsan+0x11d71fba) 10 decltype(std::__1::forward&)::$_3&>(fp)()) std::__1::__invoke&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x11d71fba) 11 void std::__1::__invoke_void_return_wrapper::__call&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x11d71fba) 12 std::__1::__function::__default_alloc_func&)::$_3, void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x11d71fba) 13 void std::__1::__function::__policy_invoker::__call_impl&)::$_3, void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x11d71fba) 14 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b3b8e0) 15 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b3b8e0) 16 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b3b8e0) 17 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3e600) 18 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(fp0)...)) std::__1::__invoke_constexpr::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3682:1 (clickhouse-tsan+0x8b3e600) 19 decltype(auto) std::__1::__apply_tuple_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::__tuple_indices&...>) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1415:1 (clickhouse-tsan+0x8b3e600) 20 decltype(auto) std::__1::apply::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1424:1 (clickhouse-tsan+0x8b3e600) 21 ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()::operator()() obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:178:13 (clickhouse-tsan+0x8b3e600) 22 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3e561) 23 void std::__1::__invoke_void_return_wrapper::__call::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x8b3e561) 24 std::__1::__function::__default_alloc_func::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x8b3e561) 25 void std::__1::__function::__policy_invoker::__call_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x8b3e561) 26 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b39205) 27 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b39205) 28 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b39205) 29 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3cea8) 30 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3cea8) 31 void std::__1::__thread_execute >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(std::__1::tuple::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>&, std::__1::__tuple_indices<>) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:280:5 (clickhouse-tsan+0x8b3cea8) 32 void* std::__1::__thread_proxy >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()> >(void*) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:291:5 (clickhouse-tsan+0x8b3cea8) Mutex M183938897938677368 previously acquired by the same thread here: 0 pthread_mutex_lock (clickhouse-tsan+0x8a327b6) 1 std::__1::__libcpp_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:405:10 (clickhouse-tsan+0x17cdb4f9) 2 std::__1::mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:33:14 (clickhouse-tsan+0x17cdb4f9) 3 std::__1::lock_guard::lock_guard(std::__1::mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse-tsan+0x12be1ca9) 4 DB::IBackgroundJobExecutor::start() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:155:21 (clickhouse-tsan+0x12be1ca9) 5 DB::StorageMergeTree::startup() obj-x86_64-linux-gnu/../src/Storages/StorageMergeTree.cpp:112:29 (clickhouse-tsan+0x129ed46e) 6 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_2::operator()(std::__1::shared_ptr const&) const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:230:16 (clickhouse-tsan+0x11d71fba) 7 DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3::operator()() const obj-x86_64-linux-gnu/../src/Databases/DatabaseOrdinary.cpp:238:56 (clickhouse-tsan+0x11d71fba) 8 decltype(std::__1::forward&)::$_3&>(fp)()) std::__1::__invoke&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x11d71fba) 9 void std::__1::__invoke_void_return_wrapper::__call&)::$_3&>(DB::DatabaseOrdinary::startupTables(ThreadPoolImpl&)::$_3&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x11d71fba) 10 std::__1::__function::__default_alloc_func&)::$_3, void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x11d71fba) 11 void std::__1::__function::__policy_invoker::__call_impl&)::$_3, void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x11d71fba) 12 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b3b8e0) 13 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b3b8e0) 14 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b3b8e0) 15 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3e600) 16 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(fp0)...)) std::__1::__invoke_constexpr::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3682:1 (clickhouse-tsan+0x8b3e600) 17 decltype(auto) std::__1::__apply_tuple_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::__tuple_indices&...>) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1415:1 (clickhouse-tsan+0x8b3e600) 18 decltype(auto) std::__1::apply::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&, std::__1::tuple<>&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&) obj-x86_64-linux-gnu/../contrib/libcxx/include/tuple:1424:1 (clickhouse-tsan+0x8b3e600) 19 ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()::operator()() obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:178:13 (clickhouse-tsan+0x8b3e600) 20 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3e561) 21 void std::__1::__invoke_void_return_wrapper::__call::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'()&>(void&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/__functional_base:348:9 (clickhouse-tsan+0x8b3e561) 22 std::__1::__function::__default_alloc_func::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()>::operator()() obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1608:12 (clickhouse-tsan+0x8b3e561) 23 void std::__1::__function::__policy_invoker::__call_impl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...)::'lambda'(), void ()> >(std::__1::__function::__policy_storage const*) obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2089:16 (clickhouse-tsan+0x8b3e561) 24 std::__1::__function::__policy_func::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2221:16 (clickhouse-tsan+0x8b39205) 25 std::__1::function::operator()() const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12 (clickhouse-tsan+0x8b39205) 26 ThreadPoolImpl::worker(std::__1::__list_iterator) obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:247:17 (clickhouse-tsan+0x8b39205) 27 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:124:73 (clickhouse-tsan+0x8b3cea8) 28 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(void&&, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()&&...) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1 (clickhouse-tsan+0x8b3cea8) 29 void std::__1::__thread_execute >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(std::__1::tuple::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>&, std::__1::__tuple_indices<>) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:280:5 (clickhouse-tsan+0x8b3cea8) 30 void* std::__1::__thread_proxy >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()> >(void*) obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:291:5 (clickhouse-tsan+0x8b3cea8) Mutex M183938897938677368 acquired here while holding mutex M2505 in main thread: 0 pthread_mutex_lock (clickhouse-tsan+0x8a327b6) 1 std::__1::__libcpp_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:405:10 (clickhouse-tsan+0x17cdb4f9) 2 std::__1::mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:33:14 (clickhouse-tsan+0x17cdb4f9) 3 std::__1::lock_guard::lock_guard(std::__1::mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse-tsan+0x12be261b) 4 DB::IBackgroundJobExecutor::finish() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:167:21 (clickhouse-tsan+0x12be261b) 5 DB::IBackgroundJobExecutor::~IBackgroundJobExecutor() obj-x86_64-linux-gnu/../src/Storages/MergeTree/BackgroundJobsExecutor.cpp:185:5 (clickhouse-tsan+0x12be261b) 6 DB::StorageMergeTree::~StorageMergeTree() obj-x86_64-linux-gnu/../src/Storages/StorageMergeTree.cpp:174:1 (clickhouse-tsan+0x129ed768) 7 std::__1::default_delete::operator()(DB::StorageMergeTree*) const obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:1397:5 (clickhouse-tsan+0x12e48b0b) 8 std::__1::__shared_ptr_pointer::__shared_ptr_default_delete, std::__1::allocator >::__on_zero_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2565:5 (clickhouse-tsan+0x12e48b0b) 9 std::__1::__shared_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2475:9 (clickhouse-tsan+0x125b53ea) 10 std::__1::__shared_weak_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2517:27 (clickhouse-tsan+0x125b53ea) 11 std::__1::shared_ptr::~shared_ptr() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3212:19 (clickhouse-tsan+0x125b53ea) 12 DB::SystemLog::~SystemLog() obj-x86_64-linux-gnu/../src/Interpreters/SystemLog.h:118:7 (clickhouse-tsan+0x125b53ea) 13 std::__1::allocator::destroy(DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:891:15 (clickhouse-tsan+0x125b1dd8) 14 void std::__1::allocator_traits >::__destroy(std::__1::integral_constant, std::__1::allocator&, DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:539:21 (clickhouse-tsan+0x125b1dd8) 15 void std::__1::allocator_traits >::destroy(std::__1::allocator&, DB::AsynchronousMetricLog*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__memory/allocator_traits.h:487:14 (clickhouse-tsan+0x125b1dd8) 16 std::__1::__shared_ptr_emplace >::__on_zero_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2611:9 (clickhouse-tsan+0x125b1dd8) 17 std::__1::__shared_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2475:9 (clickhouse-tsan+0x125904ff) 18 std::__1::__shared_weak_count::__release_shared() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2517:27 (clickhouse-tsan+0x125904ff) 19 std::__1::shared_ptr::~shared_ptr() obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3212:19 (clickhouse-tsan+0x125904ff) 20 DB::SystemLogs::~SystemLogs() obj-x86_64-linux-gnu/../src/Interpreters/SystemLog.cpp:155:1 (clickhouse-tsan+0x125904ff) 21 std::__1::__optional_destruct_base::reset() obj-x86_64-linux-gnu/../contrib/libcxx/include/optional:245:21 (clickhouse-tsan+0x11e43655) 22 DB::ContextShared::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:443:21 (clickhouse-tsan+0x11e43655) 23 DB::Context::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:2251:13 (clickhouse-tsan+0x11e3be37) 24 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:892:5 (clickhouse-tsan+0x8abacc2) 25 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::invoke() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:97:9 (clickhouse-tsan+0x8abacc2) 26 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::~basic_scope_guard() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:47:28 (clickhouse-tsan+0x8abacc2) 27 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:1395:1 (clickhouse-tsan+0x8ab5cba) 28 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b4ac7b) 29 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa97be) 30 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b67093) 31 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8aa838e) 32 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8aa68f9) Mutex M2505 previously acquired by the same thread here: 0 pthread_mutex_lock (clickhouse-tsan+0x8a327b6) 1 std::__1::__libcpp_recursive_mutex_lock(pthread_mutex_t*) obj-x86_64-linux-gnu/../contrib/libcxx/include/__threading_support:385:10 (clickhouse-tsan+0x17cdb689) 2 std::__1::recursive_mutex::lock() obj-x86_64-linux-gnu/../contrib/libcxx/src/mutex.cpp:71:14 (clickhouse-tsan+0x17cdb689) 3 std::__1::lock_guard::lock_guard(std::__1::recursive_mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse-tsan+0x11e4363f) 4 DB::ContextShared::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:438:21 (clickhouse-tsan+0x11e4363f) 5 DB::Context::shutdown() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:2251:13 (clickhouse-tsan+0x11e3be37) 6 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:892:5 (clickhouse-tsan+0x8abacc2) 7 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::invoke() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:97:9 (clickhouse-tsan+0x8abacc2) 8 ext::basic_scope_guard, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&)::$_5>::~basic_scope_guard() obj-x86_64-linux-gnu/../base/common/../ext/scope_guard.h:47:28 (clickhouse-tsan+0x8abacc2) 9 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:1395:1 (clickhouse-tsan+0x8ab5cba) 10 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse-tsan+0x15b4ac7b) 11 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:342:25 (clickhouse-tsan+0x8aa97be) 12 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse-tsan+0x15b67093) 13 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:134:20 (clickhouse-tsan+0x8aa838e) 14 main obj-x86_64-linux-gnu/../programs/main.cpp:368:12 (clickhouse-tsan+0x8aa68f9) [1]: https://clickhouse-test-reports.s3.yandex.net/21318/f3b1ad0f5d1024275674e1beac24251ae97c8453/functional_stateful_tests_(thread).html#fail1 v2: Convert ContextSharedPart::system_logs to std::unique_ptr (to avoid copy ctor) v3: Fix readability-identifier-naming,-warnings-as-errors for system_logs_ v4: fix conflicts --- src/Interpreters/Context.cpp | 44 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index ab327089333..6eaefa0978e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -373,7 +373,7 @@ struct ContextSharedPart std::atomic_size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default) String format_schema_path; /// Path to a directory that contains schema files used by input formats. ActionLocksManagerPtr action_locks_manager; /// Set of storages' action lockers - std::optional system_logs; /// Used to log queries and operations on parts + std::unique_ptr system_logs; /// Used to log queries and operations on parts std::optional storage_s3_settings; /// Settings of S3 storage RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml @@ -442,26 +442,32 @@ struct ContextSharedPart DatabaseCatalog::shutdown(); - auto lock = std::lock_guard(mutex); + std::unique_ptr delete_system_logs; + { + auto lock = std::lock_guard(mutex); - /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). - /// TODO: Get rid of this. + /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). + /// TODO: Get rid of this. - system_logs.reset(); - embedded_dictionaries.reset(); - external_dictionaries_loader.reset(); - models_repository_guard.reset(); - external_models_loader.reset(); - buffer_flush_schedule_pool.reset(); - schedule_pool.reset(); - distributed_schedule_pool.reset(); - message_broker_schedule_pool.reset(); - ddl_worker.reset(); + delete_system_logs = std::move(system_logs); + embedded_dictionaries.reset(); + external_dictionaries_loader.reset(); + models_repository_guard.reset(); + external_models_loader.reset(); + buffer_flush_schedule_pool.reset(); + schedule_pool.reset(); + distributed_schedule_pool.reset(); + message_broker_schedule_pool.reset(); + ddl_worker.reset(); - /// Stop trace collector if any - trace_collector.reset(); - /// Stop zookeeper connection - zookeeper.reset(); + /// Stop trace collector if any + trace_collector.reset(); + /// Stop zookeeper connection + zookeeper.reset(); + } + + /// Can be removed w/o context lock + delete_system_logs.reset(); } bool hasTraceCollector() const @@ -1912,7 +1918,7 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptrsystem_logs.emplace(getGlobalContext(), getConfigRef()); + shared->system_logs = std::make_unique(getGlobalContext(), getConfigRef()); } void Context::initializeTraceCollector() From c849686b16f92840cab2b1099ebf87961ad22a90 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 13 Apr 2021 08:20:59 +0300 Subject: [PATCH 5/6] Fix current connections count with shutdown_wait_unfinished=0 --- programs/server/Server.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index cb53687df4d..e3b4316079c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -173,18 +173,24 @@ int waitServersToFinish(std::vector & servers, size_t const int sleep_one_ms = 100; int sleep_current_ms = 0; int current_connections = 0; - while (sleep_current_ms < sleep_max_ms) + for (;;) { current_connections = 0; + for (auto & server : servers) { server.stop(); current_connections += server.currentConnections(); } + if (!current_connections) break; + sleep_current_ms += sleep_one_ms; - std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms)); + if (sleep_current_ms < sleep_max_ms) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms)); + else + break; } return current_connections; } From 3afa94612a17d7a9501a37429c51a377d392ced5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 28 Feb 2021 22:31:27 +0300 Subject: [PATCH 6/6] Add a test to ensure that server will wait the server thread pools v2: add into skip_list v3: print server log on error v4: increase sleep time to trigger some issues under TSAN v5: avoid ports overlaps v6: avoid endless loops to print server log on failure --- ...se_server_wait_server_pool_long.config.xml | 35 ++++++++ ...use_server_wait_server_pool_long.reference | 0 ...clickhouse_server_wait_server_pool_long.sh | 83 +++++++++++++++++++ tests/queries/shell_config.sh | 7 ++ tests/queries/skip_list.json | 1 + 5 files changed, 126 insertions(+) create mode 100644 tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.config.xml create mode 100644 tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.reference create mode 100755 tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.sh diff --git a/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.config.xml b/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.config.xml new file mode 100644 index 00000000000..2d0a480a375 --- /dev/null +++ b/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.config.xml @@ -0,0 +1,35 @@ + + + + trace + true + + + 9000 + + ./ + + 0 + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.reference b/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.sh b/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.sh new file mode 100755 index 00000000000..a4fd7529ab2 --- /dev/null +++ b/tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +server_opts=( + "--config-file=$CUR_DIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml" + "--" + # to avoid multiple listen sockets (complexity for port discovering) + "--listen_host=127.1" + # we will discover the real port later. + "--tcp_port=0" + "--shutdown_wait_unfinished=0" +) +CLICKHOUSE_WATCHDOG_ENABLE=0 $CLICKHOUSE_SERVER_BINARY "${server_opts[@]}" >& clickhouse-server.log & +server_pid=$! + +trap cleanup EXIT +function cleanup() +{ + kill -9 $server_pid + kill -9 $client_pid + + echo "Test failed. Server log:" + cat clickhouse-server.log + rm -f clickhouse-server.log + + exit 1 +} + +server_port= +i=0 retries=300 +# wait until server will start to listen (max 30 seconds) +while [[ -z $server_port ]] && [[ $i -lt $retries ]]; do + server_port=$(lsof -n -a -P -i tcp -s tcp:LISTEN -p $server_pid 2>/dev/null | awk -F'[ :]' '/LISTEN/ { print $(NF-1) }') + ((++i)) + sleep 0.1 +done +if [[ -z $server_port ]]; then + echo "Cannot wait for LISTEN socket" >&2 + exit 1 +fi + +# wait for the server to start accepting tcp connections (max 30 seconds) +i=0 retries=300 +while ! $CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" --format Null -q 'select 1' 2>/dev/null && [[ $i -lt $retries ]]; do + sleep 0.1 +done +if ! $CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" --format Null -q 'select 1'; then + echo "Cannot wait until server will start accepting connections on " >&2 + exit 1 +fi + +query_id="$CLICKHOUSE_DATABASE-$SECONDS" +$CLICKHOUSE_CLIENT_BINARY --query_id "$query_id" --host 127.1 --port "$server_port" --format Null -q 'select sleepEachRow(1) from numbers(10)' 2>/dev/null & +client_pid=$! + +# wait until the query will appear in processlist (max 10 second) +# (it is enough to trigger the problem) +i=0 retries=1000 +while [[ $($CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" -q "select count() from system.processes where query_id = '$query_id'") != "1" ]] && [[ $i -lt $retries ]]; do + sleep 0.01 +done +if [[ $($CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" -q "select count() from system.processes where query_id = '$query_id'") != "1" ]]; then + echo "Cannot wait until the query will start" >&2 + exit 1 +fi + +# send TERM and save the error code to ensure that it is 0 (EXIT_SUCCESS) +kill $server_pid +wait $server_pid +return_code=$? + +wait $client_pid + +trap '' EXIT +if [ $return_code != 0 ]; then + cat clickhouse-server.log +fi +rm -f clickhouse-server.log + +exit $return_code diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 5b942a95d02..ff8de80c0cc 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -23,14 +23,21 @@ export CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DA [ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} +# client [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} [ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client} export CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} export CLICKHOUSE_CLIENT_OPT="${CLICKHOUSE_CLIENT_OPT0:-} ${CLICKHOUSE_CLIENT_OPT:-}" export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT:-}"} +# local [ -x "${CLICKHOUSE_BINARY}-local" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"} export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} +# server +[ -x "${CLICKHOUSE_BINARY}-server" ] && CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY}-server"} +[ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY} server"} +export CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY}-server"} +# others export CLICKHOUSE_OBFUSCATOR=${CLICKHOUSE_OBFUSCATOR:="${CLICKHOUSE_BINARY}-obfuscator"} export CLICKHOUSE_COMPRESSOR=${CLICKHOUSE_COMPRESSOR:="${CLICKHOUSE_BINARY}-compressor"} export CLICKHOUSE_BENCHMARK=${CLICKHOUSE_BENCHMARK:="${CLICKHOUSE_BINARY}-benchmark ${CLICKHOUSE_BENCHMARK_OPT0:-}"} diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index d41a41bd524..740a8d33ae7 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -696,6 +696,7 @@ "01682_cache_dictionary_complex_key", "01684_ssd_cache_dictionary_simple_key", "01685_ssd_cache_dictionary_complex_key", + "01737_clickhouse_server_wait_server_pool_long", // This test is fully compatible to run in parallel, however under ASAN processes are pretty heavy and may fail under flaky adress check. "01760_system_dictionaries", "01760_polygon_dictionaries", "01778_hierarchical_dictionaries",