mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 16:42:05 +00:00
e6e8b065bc
ClickHouse uses JEMALLOC_PROF_LIBGCC - _Unwind_Backtrace() for jemalloc,
which calls _Unwind_Backtrace() during bootstrap of jemalloc, and if,
dlsym() has allocations it will lead to deadlock:
0 __lll_lock_wait (futex=futex@entry=0x16516410 <init_lock+64>, private=0) at lowlevellock.c:52
1 0x00007ffff7f9d0a3 in __GI___pthread_mutex_lock (mutex=0x16516410 <init_lock+64>) at ../nptl/pthread_mutex_lock.c:80
2 0x000000000ba6ec23 in pthread_mutex_lock (arg=arg@entry=0x16516410 <init_lock+64>) at ./build/RelWithDebInfo/./src/Common/ThreadFuzzer.cpp:445
3 0x0000000014215fd4 in malloc_mutex_lock_final (mutex=0x165163d0 <init_lock>) at ./contrib/jemalloc/include/jemalloc/internal/mutex.h:151
4 malloc_mutex_lock_slow (mutex=0x165163d0 <init_lock>) at ./build/RelWithDebInfo/./contrib/jemalloc/src/mutex.c:90
5 0x00000000141defc6 in malloc_mutex_lock (tsdn=0x0, mutex=<optimized out>) at ./contrib/jemalloc/include/jemalloc/internal/mutex.h:217
6 malloc_init_hard () at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:2118
7 0x00000000141d05db in malloc_init () at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:298
8 imalloc_init_check (sopts=<optimized out>, dopts=<optimized out>) at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:2658
9 imalloc (sopts=<optimized out>, dopts=<optimized out>) at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:2689
10 calloc (num=num@entry=1, size=size@entry=32) at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:2852
11 0x00007ffff7c42c05 in _dlerror_run (operate=operate@entry=0x7ffff7c42490 <dlsym_doit>, args=args@entry=0x7fffffffda40) at dlerror.c:148
12 0x00007ffff7c42525 in __dlsym (handle=<optimized out>, name=0xe7f568 "dl_iterate_phdr") at dlsym.c:70
13 0x000000001408e1a0 in (anonymous namespace)::getOriginalDLIteratePHDR () at ./build/RelWithDebInfo/./base/base/phdr_cache.cpp:44
14 dl_iterate_phdr (callback=0x16287a60 <libunwind::findUnwindSectionsByPhdr(dl_phdr_info*, unsigned long, void*)>, data=0x7fffffffdb58) at ./build/RelWithDebInfo/./base/base/phdr_cache.cpp:64
15 0x0000000016288468 in libunwind::LocalAddressSpace::findUnwindSections (this=<optimized out>, targetAddr=371765377, info=...) at ./contrib/libunwind/src/AddressSpace.hpp:605
16 libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_x86_64>::setInfoBasedOnIPRegister (this=this@entry=0x7fffffffde70, isReturnAddress=false) at ./contrib/libunwind/src/UnwindCursor.hpp:2553
17 0x0000000016287617 in __unw_init_local (cursor=cursor@entry=0x7fffffffde70, context=context@entry=0x7fffffffddc8) at ./build/RelWithDebInfo/./contrib/libunwind/src/libunwind.cpp:91
18 0x000000001628b094 in _Unwind_Backtrace (callback=0x14226f40 <prof_unwind_init_callback>, ref=0x0) at ./build/RelWithDebInfo/./contrib/libunwind/src/UnwindLevel1-gcc-ext.c:106
19 0x000000001421a7e0 in prof_boot2 (tsd=tsd@entry=0x7ffff7c357c0, base=<optimized out>) at ./build/RelWithDebInfo/./contrib/jemalloc/src/prof.c:693
20 0x00000000141df3e7 in malloc_init_hard () at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:2151
21 0x00000000141d96f1 in malloc_init () at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:298
22 nallocx (size=32, flags=0) at ./build/RelWithDebInfo/./contrib/jemalloc/src/jemalloc.c:4011
23 0x000000000b9fdd9b in _ZN6Memory23getActualAllocationSizeITpTkNSt3__17same_asISt11align_val_tEEJEQsr2DBE16OptionalArgumentIDpT_EEEmmS5_ (size=32) at ./src/Common/memory.h:197
24 _ZN6Memory11trackMemoryITpTkNSt3__17same_asISt11align_val_tEEJEQsr2DBE16OptionalArgumentIDpT_EEEmmR15AllocationTraceS5_ (size=32, trace=...) at ./src/Common/memory.h:208
25 operator new (size=size@entry=32) at ./build/RelWithDebInfo/./src/Common/new_delete.cpp:53
26 0x000000001540e0a6 in google::protobuf::internal::ShutdownData::get () at ./build/RelWithDebInfo/./contrib/google-protobuf/src/google/protobuf/message_lite.cc:679
27 google::protobuf::internal::OnShutdownRun (f=0x15407420 <google::protobuf::internal::DestroyString(void const*)>, arg=0x167a6368 <google::protobuf::internal::fixed_address_empty_string>) at ./build/RelWithDebInfo/./contrib/google-protobuf/src/google/protobuf/message_lite.cc:697
28 0x0000000015407560 in global constructors keyed to 000101 () at ./contrib/google-protobuf/src/google/protobuf/generated_message_util.h:180
29 0x000000001629101d in __libc_csu_init ()
30 0x00007ffff7dba010 in __libc_start_main (main=0x66e3b00 <main(int, char**)>, argc=4, argv=0x7fffffffe258, init=0x16290fd0 <__libc_csu_init>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fffffffe248) at ../csu/libc-start.c:264
31 0x00000000066e302e in _start ()
And this is indeed what happens for glibc prio 2.34, since only in [2]
the allocation had been removed
[2]: https://sourceware.org/git/?p=glibc.git;a=commit;h=fada9018199c21c469ff0e731ef75c6020074ac9
But, jemalloc provides also JEMALLOC_PROF_LIBUNWIND, yes it is not llvm
libunwind, but gnu one, but ClickHouse version of llvm libunwind
provides unw_backtrace(), so we can use it, and it will not be called
during jemalloc bootstrap, so deadlock should not be possible.
Funny thing that for this deadlock the workaround is to enable
background_thread, because a thread created for it will initialize TLS for
__dlerror().
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
|
||
|---|---|---|
| .. | ||
| abseil-cpp@a3c4dd3e77 | ||
| abseil-cpp-cmake | ||
| aklomp-base64@e77bd70bdd | ||
| aklomp-base64-cmake | ||
| AMQP-CPP@00f09897ce | ||
| amqpcpp-cmake | ||
| annoy@f2ac8e7b48 | ||
| annoy-cmake | ||
| arrow@5cfccd8ea6 | ||
| arrow-cmake | ||
| avro@545e700268 | ||
| avro-cmake | ||
| aws@1c2946bfcb | ||
| aws-c-auth@baeffa791d | ||
| aws-c-cal@1586846816 | ||
| aws-c-common@80f21b3cac | ||
| aws-c-compression@99ec79ee29 | ||
| aws-c-event-stream@08f24e384e | ||
| aws-c-http@a082f8a206 | ||
| aws-c-io@11ce3c750a | ||
| aws-c-mqtt@6d36cd3726 | ||
| aws-c-s3@de36fee8fe | ||
| aws-c-sdkutils@fd8c0ba2e2 | ||
| aws-checksums@321b805559 | ||
| aws-cmake | ||
| aws-crt-cpp@f532d6abc0 | ||
| aws-s2n-tls@9a1e754540 | ||
| azure@92c94d7f37 | ||
| azure-cmake | ||
| boost@ae94606a70 | ||
| boost-cmake | ||
| brotli@63be8a9940 | ||
| brotli-cmake | ||
| bzip2@bf905ea225 | ||
| bzip2-cmake | ||
| c-ares@6360e96b5c | ||
| c-ares-cmake | ||
| capnproto@976209a6d1 | ||
| capnproto-cmake | ||
| cassandra@f4a31e92a2 | ||
| cassandra-cmake | ||
| cctz@7918cb7afe | ||
| cctz-cmake | ||
| cityhash102 | ||
| cld2@217ba8b880 | ||
| cld2-cmake | ||
| consistent-hashing | ||
| corrosion@d5bdbfacb4 | ||
| corrosion-cmake | ||
| cppkafka@9c5ea0e332 | ||
| cppkafka-cmake | ||
| crc32-s390x@30980583bf | ||
| crc32-s390x-cmake | ||
| crc32-vpmsum@4521554393 | ||
| crc32-vpmsum-cmake | ||
| croaring@9b7cc0ff1c | ||
| croaring-cmake | ||
| curl@de7b3e8921 | ||
| curl-cmake | ||
| cyrus-sasl@e6466edfd6 | ||
| cyrus-sasl-cmake | ||
| datasketches-cpp@76edd74f5d | ||
| datasketches-cpp-cmake | ||
| double-conversion@4f7a25d8ce | ||
| double-conversion-cmake | ||
| dragonbox@923705af6f | ||
| dragonbox-cmake | ||
| expected@3f0ca7b192 | ||
| expected-cmake | ||
| fast_float@7eae925b51 | ||
| fast_float-cmake | ||
| fastops@1460583af7 | ||
| fastops-cmake | ||
| flatbuffers@eb3f827948 | ||
| fmtlib@a33701196a | ||
| fmtlib-cmake | ||
| FP16@0a92994d72 | ||
| FP16-cmake | ||
| google-benchmark@2257fa4d6a | ||
| google-benchmark-cmake | ||
| google-protobuf@0fae801fb4 | ||
| google-protobuf-cmake | ||
| googletest@a7f443b80b | ||
| googletest-cmake | ||
| grpc@f5b7fdc2df | ||
| grpc-cmake | ||
| gwpasan-cmake | ||
| h3@c7f46cfd71 | ||
| h3-cmake | ||
| hive-metastore@809a77d435 | ||
| hive-metastore-cmake | ||
| icu@a56dde820d | ||
| icu-cmake | ||
| icudata@c8e717892a | ||
| idna@3c8be01d42 | ||
| idna-cmake | ||
| idxd-config@a836ce0e42 | ||
| idxd-config-cmake | ||
| incbin@6e576cae5a | ||
| incbin-cmake | ||
| isa-l@9f2b68f057 | ||
| isa-l-cmake | ||
| jemalloc@41a859ef73 | ||
| jemalloc-cmake | ||
| krb5@71b06c2276 | ||
| krb5-cmake | ||
| lemmagen-c@59537bdcf5 | ||
| lemmagen-c-cmake | ||
| libarchive@ee45796171 | ||
| libarchive-cmake | ||
| libbcrypt@8aa32ad94e | ||
| libbcrypt-cmake | ||
| libcpuid@503083acb7 | ||
| libcpuid-cmake | ||
| libcxx-cmake | ||
| libcxxabi-cmake | ||
| libdivide@3bd3438857 | ||
| libdivide-cmake | ||
| libfarmhash | ||
| libfiu@b85edbde4c | ||
| libfiu-cmake | ||
| libfuzzer-cmake | ||
| libgsasl@0fb79e7609 | ||
| libgsasl-cmake | ||
| libhdfs3@0d04201c45 | ||
| libhdfs3-cmake | ||
| libmetrohash | ||
| libpq@2446f2c856 | ||
| libpq-cmake | ||
| libpqxx@c995193a3a | ||
| libpqxx-cmake | ||
| libprotobuf-mutator@a304ec48dc | ||
| libprotobuf-mutator-cmake | ||
| librdkafka@2d2aab6f5b | ||
| librdkafka-cmake | ||
| libssh@ed4011b918 | ||
| libssh-cmake | ||
| libstemmer_c@c753054304 | ||
| libstemmer-c-cmake | ||
| libunwind@d6a01c4632 | ||
| libunwind-cmake | ||
| liburing@f4e42a515c | ||
| liburing-cmake | ||
| libuv@4482964660 | ||
| libuv-cmake | ||
| libxml2@223cb03a5d | ||
| libxml2-cmake | ||
| llvm-project@2a8967b60c | ||
| llvm-project-cmake | ||
| lz4@145f3804ca | ||
| lz4-cmake | ||
| magic_enum@38f86e4d09 | ||
| magic-enum-cmake | ||
| mariadb-connector-c@d0a788c5b9 | ||
| mariadb-connector-c-cmake | ||
| miniselect@be0af6bd0b | ||
| miniselect-cmake | ||
| minizip-ng@f3d400e999 | ||
| minizip-ng-cmake | ||
| morton-nd@3795491a4a | ||
| morton-nd-cmake | ||
| msgpack-c@46684265d5 | ||
| msgpack-c-cmake | ||
| murmurhash | ||
| nanodbc@df52a1232d | ||
| nanodbc-cmake | ||
| nats-io@1e2597c546 | ||
| nats-io-cmake | ||
| nlp-data@5591f91f5e | ||
| NuRaft@cb5dc3c906 | ||
| nuraft-cmake | ||
| openldap@5671b80e36 | ||
| openldap-cmake | ||
| openssl@ee2bb8513b | ||
| openssl-cmake | ||
| orc@bcc025c098 | ||
| pdqsort | ||
| pdqsort-cmake | ||
| pocketfft@9efd4da52c | ||
| pocketfft-cmake | ||
| prometheus-protobufs | ||
| prometheus-protobufs-cmake | ||
| prometheus-protobufs-gogo | ||
| QAT-ZSTD-Plugin@e5a134e12d | ||
| QAT-ZSTD-Plugin-cmake | ||
| qatlib@abe15d7bfc | ||
| qatlib-cmake | ||
| qpl@d4715e0e79 | ||
| qpl-cmake | ||
| rapidjson@800ca2f38f | ||
| rapidjson-cmake | ||
| re2@85dd7ad833 | ||
| re2-cmake | ||
| replxx@5d04501f93 | ||
| replxx-cmake | ||
| robin-map@851a59e0e3 | ||
| robin-map-cmake | ||
| rocksdb@078fa56386 | ||
| rocksdb-cmake | ||
| rust_vendor@08e82ca654 | ||
| s2geometry@6522a40338 | ||
| s2geometry-cmake | ||
| sentry-native@bc359f86cb | ||
| sentry-native-cmake | ||
| simdjson@6060be2fdf | ||
| simdjson-cmake | ||
| SimSIMD@de2cb75b9e | ||
| SimSIMD-cmake | ||
| snappy@6ebb5b1ab8 | ||
| snappy-cmake | ||
| sparse-checkout | ||
| sparsehash-c11@cf0bffaa45 | ||
| sparsehash-c11-cmake | ||
| sqids-cpp@a471f53672 | ||
| sqids-cpp-cmake | ||
| sqlite-amalgamation@2059807989 | ||
| sqlite-cmake | ||
| sysroot@cc385041b2 | ||
| thrift@2a93df80f2 | ||
| thrift-cmake | ||
| ulid-c@c433b6783c | ||
| ulid-c-cmake | ||
| unixodbc@18e0ebe2a1 | ||
| unixodbc-cmake | ||
| usearch@955c6f9c11 | ||
| usearch-cmake | ||
| vectorscan@d29730e1cb | ||
| vectorscan-cmake | ||
| wordnet-blast@1d16ac2803 | ||
| wordnet-blast-cmake | ||
| wyhash@991aa3dab6 | ||
| wyhash-cmake | ||
| xxHash@bbb27a5efb | ||
| xxHash-cmake | ||
| xz@869b9d1b4e | ||
| xz-cmake | ||
| yaml-cpp@f91e938341 | ||
| yaml-cpp-cmake | ||
| zlib-ng@50f0eae1a4 | ||
| zlib-ng-cmake | ||
| zstd@63779c7982 | ||
| zstd-cmake | ||
| CMakeLists.txt | ||
| update-submodules.sh | ||