ClickHouse/tests/integration/test_send_crash_reports/test.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

74 lines
2.0 KiB
Python
Raw Normal View History

# pylint: disable=redefined-outer-name
# pylint: disable=unused-argument
# pylint: disable=line-too-long
# pylint: disable=bare-except
2020-05-29 19:53:16 +00:00
import os
import time
import pytest
2020-05-29 19:53:16 +00:00
import helpers.cluster
import helpers.test_tools
2020-10-02 16:54:07 +00:00
from . import fake_sentry_server
2020-05-29 19:53:16 +00:00
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.fixture(scope="module")
def started_node():
cluster = helpers.cluster.ClickHouseCluster(__file__)
try:
node = cluster.add_instance(
"node",
main_configs=[
os.path.join(SCRIPT_DIR, "configs", "config_send_crash_reports.xml")
],
)
cluster.start()
yield node
finally:
# It will print Fatal message after pkill -SEGV, suppress it
try:
cluster.shutdown()
except:
pass
2020-05-29 19:53:16 +00:00
2021-04-25 02:25:46 +00:00
def test_send_segfault(started_node):
tests: skip test_send_crash_reports under ASan Since it also does not fits into timeouts [1]: 2022-09-08 21:16:20 [ 377 ] DEBUG : Command:['docker', 'exec', '-u', 'root', 'roottestsendcrashreports_node_1', 'bash', '-c', 'pkill -SEGV clickhouse'] (cluster.py:95, run_and_check) ... 2022-09-08 21:16:22 [ 377 ] DEBUG : run container_id:roottestsendcrashreports_node_1 detach:False nothrow:False cmd: ['cat', '/result.txt'] (cluster.py:1744, exec_in_container) ... 2022-09-08 21:16:36 [ 377 ] DEBUG : Stdout:INITIAL_STATE (cluster.py:103, run_and_check) And server logs: 2022.09.08 21:16:21.112076 [ 228 ] {} <Fatal> BaseDaemon: ######################################## 2022.09.08 21:16:21.112170 [ 228 ] {} <Fatal> BaseDaemon: (version 22.9.1.1, build id: 0F7336E0A4D64134C51C8365DADCB78A9B39AA3B) (from thread 1) (no query) Received signal Segmentation fault (11) 2022.09.08 21:16:21.112244 [ 228 ] {} <Fatal> BaseDaemon: Address: 0xde Access: read. Unknown si_code. 2022.09.08 21:16:21.112321 [ 228 ] {} <Fatal> BaseDaemon: Stack trace: 0x7fbe21d09376 0x40a4f71a 0xe293a4b 0xdc5c51a 0x38dee227 0xdc326a0 0x38e319d9 0xdc2b4e2 0xdc25fdb 0x7fbe21b2c083 0xdb636ae 2022.09.08 21:16:21.112419 [ 228 ] {} <Fatal> BaseDaemon: 3. pthread_cond_wait @ 0x7fbe21d09376 in ? 2022.09.08 21:16:21.122914 [ 228 ] {} <Fatal> BaseDaemon: 4. ./build_docker/../contrib/libcxx/src/condition_variable.cpp:0: std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) @ 0x40a4f71a in /usr/bin/clickhouse 2022.09.08 21:16:21.233016 [ 228 ] {} <Fatal> BaseDaemon: 5.1. inlined from ./build_docker/../contrib/libcxx/include/atomic:952: unsigned long std::__1::__cxx_atomic_load<unsigned long>(std::__1::__cxx_atomic_base_impl<unsigned long> const*, std::__1::memory_order) 2022.09.08 21:16:21.233135 [ 228 ] {} <Fatal> BaseDaemon: 5.2. inlined from ../contrib/libcxx/include/atomic:1582: std::__1::__atomic_base<unsigned long, false>::load(std::__1::memory_order) const 2022.09.08 21:16:21.233183 [ 228 ] {} <Fatal> BaseDaemon: 5.3. inlined from ../contrib/libcxx/include/atomic:1586: std::__1::__atomic_base<unsigned long, false>::operator unsigned long() const 2022.09.08 21:16:21.233234 [ 228 ] {} <Fatal> BaseDaemon: 5.4. inlined from ../src/Daemon/BaseDaemon.cpp:967: operator() 2022.09.08 21:16:21.233303 [ 228 ] {} <Fatal> BaseDaemon: 5.5. inlined from ../contrib/libcxx/include/__mutex_base:402: void std::__1::condition_variable::wait<BaseDaemon::waitForTerminationRequest()::$_0>(std::__1::unique_lock<std::__1::mutex>&, BaseDaemon::waitForTerminationRequest()::$_0) 2022.09.08 21:16:21.233334 [ 228 ] {} <Fatal> BaseDaemon: 5. ../src/Daemon/BaseDaemon.cpp:967: BaseDaemon::waitForTerminationRequest() @ 0xe293a4b in /usr/bin/clickhouse 2022.09.08 21:16:21.350675 [ 228 ] {} <Fatal> BaseDaemon: 6. ./build_docker/../programs/server/Server.cpp:0: DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0xdc5c51a in /usr/bin/clickhouse 2022.09.08 21:16:21.394092 [ 228 ] {} <Fatal> BaseDaemon: 7. ./build_docker/../contrib/poco/Util/src/Application.cpp:0: Poco::Util::Application::run() @ 0x38dee227 in /usr/bin/clickhouse 2022.09.08 21:16:21.654195 [ 228 ] {} <Fatal> BaseDaemon: 8. ./build_docker/../programs/server/Server.cpp:466: DB::Server::run() @ 0xdc326a0 in /usr/bin/clickhouse 2022.09.08 21:16:21.666991 [ 228 ] {} <Fatal> BaseDaemon: 9. ./build_docker/../contrib/poco/Util/src/ServerApplication.cpp:0: Poco::Util::ServerApplication::run(int, char**) @ 0x38e319d9 in /usr/bin/clickhouse 2022.09.08 21:16:21.916078 [ 228 ] {} <Fatal> BaseDaemon: 10. ./build_docker/../programs/server/Server.cpp:0: mainEntryClickHouseServer(int, char**) @ 0xdc2b4e2 in /usr/bin/clickhouse 2022.09.08 21:16:21.929922 [ 228 ] {} <Fatal> BaseDaemon: 11. ./build_docker/../programs/main.cpp:0: main @ 0xdc25fdb in /usr/bin/clickhouse 2022.09.08 21:16:21.929981 [ 228 ] {} <Fatal> BaseDaemon: 12. __libc_start_main @ 0x7fbe21b2c083 in ? 2022.09.08 21:16:30.357032 [ 228 ] {} <Fatal> BaseDaemon: 13. _start @ 0xdb636ae in /usr/bin/clickhouse 2022.09.08 21:16:31.383233 [ 228 ] {} <Fatal> BaseDaemon: Integrity check of the executable skipped because the reference checksum could not be read. (calculated checksum: 6200AC7C1270DC293DF3302E1C64399B) ... 2022.09.08 21:16:40.564453 [ 228 ] {} <Information> SentryWriter: Sending crash report [1]: https://s3.amazonaws.com/clickhouse-test-reports/41046/a0b85eaca8d4003c9fbc4571b30830d30f1984e9/integration_tests__asan__[3/3].html Though another option is to increase waiting time. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-09-09 13:09:35 +00:00
# NOTE: another option is to increase waiting time.
2021-05-31 14:30:02 +00:00
if (
started_node.is_built_with_thread_sanitizer()
tests: skip test_send_crash_reports under ASan Since it also does not fits into timeouts [1]: 2022-09-08 21:16:20 [ 377 ] DEBUG : Command:['docker', 'exec', '-u', 'root', 'roottestsendcrashreports_node_1', 'bash', '-c', 'pkill -SEGV clickhouse'] (cluster.py:95, run_and_check) ... 2022-09-08 21:16:22 [ 377 ] DEBUG : run container_id:roottestsendcrashreports_node_1 detach:False nothrow:False cmd: ['cat', '/result.txt'] (cluster.py:1744, exec_in_container) ... 2022-09-08 21:16:36 [ 377 ] DEBUG : Stdout:INITIAL_STATE (cluster.py:103, run_and_check) And server logs: 2022.09.08 21:16:21.112076 [ 228 ] {} <Fatal> BaseDaemon: ######################################## 2022.09.08 21:16:21.112170 [ 228 ] {} <Fatal> BaseDaemon: (version 22.9.1.1, build id: 0F7336E0A4D64134C51C8365DADCB78A9B39AA3B) (from thread 1) (no query) Received signal Segmentation fault (11) 2022.09.08 21:16:21.112244 [ 228 ] {} <Fatal> BaseDaemon: Address: 0xde Access: read. Unknown si_code. 2022.09.08 21:16:21.112321 [ 228 ] {} <Fatal> BaseDaemon: Stack trace: 0x7fbe21d09376 0x40a4f71a 0xe293a4b 0xdc5c51a 0x38dee227 0xdc326a0 0x38e319d9 0xdc2b4e2 0xdc25fdb 0x7fbe21b2c083 0xdb636ae 2022.09.08 21:16:21.112419 [ 228 ] {} <Fatal> BaseDaemon: 3. pthread_cond_wait @ 0x7fbe21d09376 in ? 2022.09.08 21:16:21.122914 [ 228 ] {} <Fatal> BaseDaemon: 4. ./build_docker/../contrib/libcxx/src/condition_variable.cpp:0: std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) @ 0x40a4f71a in /usr/bin/clickhouse 2022.09.08 21:16:21.233016 [ 228 ] {} <Fatal> BaseDaemon: 5.1. inlined from ./build_docker/../contrib/libcxx/include/atomic:952: unsigned long std::__1::__cxx_atomic_load<unsigned long>(std::__1::__cxx_atomic_base_impl<unsigned long> const*, std::__1::memory_order) 2022.09.08 21:16:21.233135 [ 228 ] {} <Fatal> BaseDaemon: 5.2. inlined from ../contrib/libcxx/include/atomic:1582: std::__1::__atomic_base<unsigned long, false>::load(std::__1::memory_order) const 2022.09.08 21:16:21.233183 [ 228 ] {} <Fatal> BaseDaemon: 5.3. inlined from ../contrib/libcxx/include/atomic:1586: std::__1::__atomic_base<unsigned long, false>::operator unsigned long() const 2022.09.08 21:16:21.233234 [ 228 ] {} <Fatal> BaseDaemon: 5.4. inlined from ../src/Daemon/BaseDaemon.cpp:967: operator() 2022.09.08 21:16:21.233303 [ 228 ] {} <Fatal> BaseDaemon: 5.5. inlined from ../contrib/libcxx/include/__mutex_base:402: void std::__1::condition_variable::wait<BaseDaemon::waitForTerminationRequest()::$_0>(std::__1::unique_lock<std::__1::mutex>&, BaseDaemon::waitForTerminationRequest()::$_0) 2022.09.08 21:16:21.233334 [ 228 ] {} <Fatal> BaseDaemon: 5. ../src/Daemon/BaseDaemon.cpp:967: BaseDaemon::waitForTerminationRequest() @ 0xe293a4b in /usr/bin/clickhouse 2022.09.08 21:16:21.350675 [ 228 ] {} <Fatal> BaseDaemon: 6. ./build_docker/../programs/server/Server.cpp:0: DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0xdc5c51a in /usr/bin/clickhouse 2022.09.08 21:16:21.394092 [ 228 ] {} <Fatal> BaseDaemon: 7. ./build_docker/../contrib/poco/Util/src/Application.cpp:0: Poco::Util::Application::run() @ 0x38dee227 in /usr/bin/clickhouse 2022.09.08 21:16:21.654195 [ 228 ] {} <Fatal> BaseDaemon: 8. ./build_docker/../programs/server/Server.cpp:466: DB::Server::run() @ 0xdc326a0 in /usr/bin/clickhouse 2022.09.08 21:16:21.666991 [ 228 ] {} <Fatal> BaseDaemon: 9. ./build_docker/../contrib/poco/Util/src/ServerApplication.cpp:0: Poco::Util::ServerApplication::run(int, char**) @ 0x38e319d9 in /usr/bin/clickhouse 2022.09.08 21:16:21.916078 [ 228 ] {} <Fatal> BaseDaemon: 10. ./build_docker/../programs/server/Server.cpp:0: mainEntryClickHouseServer(int, char**) @ 0xdc2b4e2 in /usr/bin/clickhouse 2022.09.08 21:16:21.929922 [ 228 ] {} <Fatal> BaseDaemon: 11. ./build_docker/../programs/main.cpp:0: main @ 0xdc25fdb in /usr/bin/clickhouse 2022.09.08 21:16:21.929981 [ 228 ] {} <Fatal> BaseDaemon: 12. __libc_start_main @ 0x7fbe21b2c083 in ? 2022.09.08 21:16:30.357032 [ 228 ] {} <Fatal> BaseDaemon: 13. _start @ 0xdb636ae in /usr/bin/clickhouse 2022.09.08 21:16:31.383233 [ 228 ] {} <Fatal> BaseDaemon: Integrity check of the executable skipped because the reference checksum could not be read. (calculated checksum: 6200AC7C1270DC293DF3302E1C64399B) ... 2022.09.08 21:16:40.564453 [ 228 ] {} <Information> SentryWriter: Sending crash report [1]: https://s3.amazonaws.com/clickhouse-test-reports/41046/a0b85eaca8d4003c9fbc4571b30830d30f1984e9/integration_tests__asan__[3/3].html Though another option is to increase waiting time. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-09-09 13:09:35 +00:00
or started_node.is_built_with_address_sanitizer()
2021-05-31 14:30:02 +00:00
or started_node.is_built_with_memory_sanitizer()
):
2021-02-08 15:46:48 +00:00
pytest.skip("doesn't fit in timeouts for stacktrace generation")
started_node.copy_file_to_container(
os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py"
)
2020-10-02 16:54:07 +00:00
started_node.exec_in_container(
["bash", "-c", "python3 /fake_sentry_server.py > /fake_sentry_server.log 2>&1"],
detach=True,
user="root",
)
2021-02-10 14:45:45 +00:00
time.sleep(1)
started_node.exec_in_container(
["bash", "-c", "pkill -SEGV clickhouse"], user="root"
)
2020-05-29 19:53:16 +00:00
result = None
for attempt in range(1, 6):
2021-02-10 14:45:45 +00:00
time.sleep(attempt)
result = started_node.exec_in_container(
["cat", fake_sentry_server.RESULT_PATH], user="root"
)
2020-05-29 19:53:16 +00:00
if result == "OK":
break
if result == "INITIAL_STATE":
2020-05-29 19:53:16 +00:00
continue
if result:
2020-05-29 19:53:16 +00:00
assert False, "Unexpected state: " + result
assert result == "OK", "Crash report not sent"