ClickHouse/base/daemon/BaseDaemon.cpp

1066 lines
34 KiB
C++
Raw Normal View History

#if defined(__clang__) && __clang_major__ >= 13
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
#include <daemon/BaseDaemon.h>
#include <daemon/SentryWriter.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
2020-12-22 02:57:22 +00:00
#include <sys/wait.h>
#include <sys/resource.h>
2020-12-22 03:17:31 +00:00
#if defined(__linux__)
#include <sys/prctl.h>
#endif
2019-01-19 21:05:20 +00:00
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <signal.h>
#include <cxxabi.h>
#include <unistd.h>
#include <typeinfo>
#include <iostream>
#include <fstream>
#include <sstream>
#include <memory>
2021-06-15 19:55:21 +00:00
#include <common/scope_guard.h>
#include <Poco/Observer.h>
#include <Poco/AutoPtr.h>
#include <Poco/PatternFormatter.h>
#include <Poco/Message.h>
#include <Poco/Util/Application.h>
#include <Poco/Exception.h>
#include <Poco/ErrorHandler.h>
#include <Poco/Condition.h>
#include <Poco/SyslogChannel.h>
#include <Poco/DirectoryIterator.h>
#include <common/logger_useful.h>
#include <common/ErrorHandlers.h>
#include <common/argsToConfig.h>
2020-02-02 02:35:47 +00:00
#include <common/getThreadId.h>
#include <common/coverage.h>
2020-06-20 10:15:53 +00:00
#include <common/sleep.h>
#include <IO/WriteBufferFromFile.h>
2019-08-17 20:33:50 +00:00
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Common/Exception.h>
#include <Common/PipeFDs.h>
#include <Common/StackTrace.h>
#include <Common/getMultipleKeysFromConfig.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Config/ConfigProcessor.h>
2020-07-31 14:53:41 +00:00
#include <Common/MemorySanitizer.h>
2020-06-20 09:07:05 +00:00
#include <Common/SymbolIndex.h>
#include <Common/getExecutablePath.h>
#include <Common/getHashOfLoadedBinary.h>
#include <Common/Elf.h>
2021-05-22 18:24:13 +00:00
#include <filesystem>
#if !defined(ARCADIA_BUILD)
2020-05-10 22:17:28 +00:00
# include <Common/config_version.h>
#endif
#if defined(OS_DARWIN)
2020-05-10 22:17:28 +00:00
# pragma GCC diagnostic ignored "-Wunused-macros"
# define _XOPEN_SOURCE 700 // ucontext is not available without _XOPEN_SOURCE
2019-03-27 15:42:24 +00:00
#endif
#include <ucontext.h>
2021-05-22 18:24:13 +00:00
namespace fs = std::filesystem;
2019-03-27 15:42:24 +00:00
DB::PipeFDs signal_pipe;
2018-01-17 18:06:39 +00:00
/** Reset signal handler to the default and send signal to itself.
* It's called from user signal handler to write core dump.
*/
static void call_default_signal_handler(int sig)
{
signal(sig, SIG_DFL);
2018-08-20 23:16:50 +00:00
raise(sig);
}
2019-12-23 20:19:49 +00:00
static constexpr size_t max_query_id_size = 127;
2020-05-09 23:22:51 +00:00
static const size_t signal_pipe_buf_size =
2019-12-23 20:19:49 +00:00
sizeof(int)
+ sizeof(siginfo_t)
+ sizeof(ucontext_t)
+ sizeof(StackTrace)
+ sizeof(UInt32)
+ max_query_id_size + 1 /// query_id + varint encoded length
+ sizeof(void*);
using signal_function = void(int, siginfo_t*, void*);
static void writeSignalIDtoSignalPipe(int sig)
{
2020-03-01 14:23:43 +00:00
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
2020-05-09 23:22:51 +00:00
char buf[signal_pipe_buf_size];
DB::WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
DB::writeBinary(sig, out);
out.next();
2020-03-01 14:23:43 +00:00
errno = saved_errno;
}
2018-01-17 18:06:39 +00:00
/** Signal handler for HUP / USR1 */
2020-03-18 03:27:32 +00:00
static void closeLogsSignalHandler(int sig, siginfo_t *, void *)
{
2021-01-12 14:34:50 +00:00
DENY_ALLOCATIONS_IN_SCOPE;
writeSignalIDtoSignalPipe(sig);
}
2020-03-18 03:27:32 +00:00
static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
{
2021-01-12 14:34:50 +00:00
DENY_ALLOCATIONS_IN_SCOPE;
writeSignalIDtoSignalPipe(sig);
}
/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
*/
static void signalHandler(int sig, siginfo_t * info, void * context)
{
2021-01-12 14:34:50 +00:00
DENY_ALLOCATIONS_IN_SCOPE;
2020-03-01 14:23:43 +00:00
auto saved_errno = errno; /// We must restore previous value of errno in signal handler.
2020-05-09 23:22:51 +00:00
char buf[signal_pipe_buf_size];
DB::WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
2019-06-28 18:06:38 +00:00
const ucontext_t signal_context = *reinterpret_cast<ucontext_t *>(context);
2019-07-02 00:19:56 +00:00
const StackTrace stack_trace(signal_context);
2019-03-27 20:37:39 +00:00
2020-05-30 21:57:37 +00:00
StringRef query_id = DB::CurrentThread::getQueryId(); /// This is signal safe.
2019-12-23 20:19:49 +00:00
query_id.size = std::min(query_id.size, max_query_id_size);
DB::writeBinary(sig, out);
DB::writePODBinary(*info, out);
2019-06-28 18:06:38 +00:00
DB::writePODBinary(signal_context, out);
2019-07-02 00:19:56 +00:00
DB::writePODBinary(stack_trace, out);
2020-02-02 02:35:47 +00:00
DB::writeBinary(UInt32(getThreadId()), out);
2019-12-23 20:19:49 +00:00
DB::writeStringBinary(query_id, out);
2020-06-20 11:17:15 +00:00
DB::writePODBinary(DB::current_thread, out);
out.next();
2019-12-02 17:29:19 +00:00
if (sig != SIGTSTP) /// This signal is used for debugging.
{
/// The time that is usually enough for separate thread to print info into log.
2021-01-27 13:25:46 +00:00
sleepForSeconds(20); /// FIXME: use some feedback from threads that process stacktrace
call_default_signal_handler(sig);
}
2020-03-01 14:23:43 +00:00
errno = saved_errno;
}
2020-08-01 15:54:44 +00:00
/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
__attribute__((__weak__)) void collectCrashLog(
Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);
2018-01-17 18:06:39 +00:00
/** The thread that read info about signal or std::terminate from pipe.
* On HUP / USR1, close log files (for new files to be opened later).
* On information about std::terminate, write it to log.
* On other signals, write info to log.
*/
class SignalListener : public Poco::Runnable
{
public:
enum Signals : int
{
StdTerminate = -1,
StopThread = -2,
SanitizerTrap = -3,
};
explicit SignalListener(BaseDaemon & daemon_)
2020-05-30 21:57:37 +00:00
: log(&Poco::Logger::get("BaseDaemon"))
, daemon(daemon_)
{
}
2020-05-09 23:22:51 +00:00
void run() override
{
2020-05-09 23:22:51 +00:00
char buf[signal_pipe_buf_size];
DB::ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf);
while (!in.eof())
{
int sig = 0;
DB::readBinary(sig, in);
2019-12-09 13:52:52 +00:00
// We may log some specific signals afterwards, with different log
// levels and more info, but for completeness we log all signals
// here at trace level.
2020-03-26 09:59:25 +00:00
// Don't use strsignal here, because it's not thread-safe.
2020-05-23 22:24:01 +00:00
LOG_TRACE(log, "Received signal {}", sig);
if (sig == Signals::StopThread)
{
2020-05-23 22:24:01 +00:00
LOG_INFO(log, "Stop SignalListener thread");
break;
}
else if (sig == SIGHUP || sig == SIGUSR1)
{
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Received signal to close logs.");
BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
2020-05-23 22:24:01 +00:00
LOG_INFO(log, "Opened new log file after received signal.");
}
else if (sig == Signals::StdTerminate)
{
UInt32 thread_num;
std::string message;
DB::readBinary(thread_num, in);
DB::readBinary(message, in);
onTerminate(message, thread_num);
}
else if (sig == SIGINT ||
sig == SIGQUIT ||
sig == SIGTERM)
{
daemon.handleSignal(sig);
}
else
{
siginfo_t info{};
ucontext_t context{};
2019-07-02 00:19:56 +00:00
StackTrace stack_trace(NoCapture{});
UInt32 thread_num{};
2019-12-23 20:19:49 +00:00
std::string query_id;
2020-06-20 11:17:15 +00:00
DB::ThreadStatus * thread_ptr{};
if (sig != SanitizerTrap)
{
DB::readPODBinary(info, in);
DB::readPODBinary(context, in);
}
2019-07-02 00:19:56 +00:00
DB::readPODBinary(stack_trace, in);
DB::readBinary(thread_num, in);
2019-12-23 20:19:49 +00:00
DB::readBinary(query_id, in);
2020-06-20 11:17:15 +00:00
DB::readPODBinary(thread_ptr, in);
2019-08-17 21:42:47 +00:00
/// This allows to receive more signals if failure happens inside onFault function.
/// Example: segfault while symbolizing stack trace.
std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_num, query_id, thread_ptr); }).detach();
}
}
}
private:
2020-05-30 21:57:37 +00:00
Poco::Logger * log;
BaseDaemon & daemon;
void onTerminate(std::string_view message, UInt32 thread_num) const
{
2021-07-29 14:00:33 +00:00
size_t pos = message.find('\n');
2020-06-20 09:07:05 +00:00
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) {}",
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, thread_num, message.substr(0, pos));
2021-07-30 06:21:03 +00:00
/// Print trace from std::terminate exception line-by-line to make it easy for grep.
while (pos != std::string_view::npos)
{
++pos;
size_t next_pos = message.find('\n', pos);
size_t size = next_pos;
if (next_pos != std::string_view::npos)
size = next_pos - pos;
LOG_FATAL(log, "{}", message.substr(pos, size));
pos = next_pos;
}
}
2019-12-23 20:19:49 +00:00
void onFault(
int sig,
const siginfo_t & info,
const ucontext_t & context,
const StackTrace & stack_trace,
UInt32 thread_num,
const std::string & query_id,
2020-06-20 11:17:15 +00:00
DB::ThreadStatus * thread_ptr) const
{
2020-06-20 11:17:15 +00:00
DB::ThreadStatus thread_status;
2019-12-23 20:19:49 +00:00
/// Send logs from this thread to client if possible.
/// It will allow client to see failure messages directly.
2020-06-20 11:17:15 +00:00
if (thread_ptr)
2019-12-23 20:19:49 +00:00
{
if (auto logs_queue = thread_ptr->getInternalTextLogsQueue())
DB::CurrentThread::attachInternalTextLogsQueue(logs_queue, DB::LogsLevel::trace);
2020-06-20 11:17:15 +00:00
}
2020-05-23 22:24:01 +00:00
LOG_FATAL(log, "########################################");
2019-12-23 20:19:49 +00:00
2020-06-20 09:07:05 +00:00
if (query_id.empty())
2019-12-23 20:19:49 +00:00
{
2020-06-20 09:07:05 +00:00
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
2020-07-31 14:53:41 +00:00
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
2021-01-08 05:09:30 +00:00
thread_num, strsignal(sig), sig);
2020-06-20 09:07:05 +00:00
}
else
{
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})",
2020-07-31 14:53:41 +00:00
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
2021-01-08 05:09:30 +00:00
thread_num, query_id, strsignal(sig), sig);
2019-12-23 20:19:49 +00:00
}
String error_message;
if (sig != SanitizerTrap)
error_message = signalToErrorMessage(sig, info, context);
else
error_message = "Sanitizer trap.";
LOG_FATAL(log, error_message);
if (stack_trace.getSize())
{
2020-05-12 03:36:09 +00:00
/// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace.
/// NOTE: This still require memory allocations and mutex lock inside logger.
/// BTW we can also print it to stderr using write syscalls.
2020-05-12 03:36:09 +00:00
std::stringstream bare_stacktrace;
bare_stacktrace << "Stack trace:";
for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
bare_stacktrace << ' ' << stack_trace.getFramePointers()[i];
2020-05-23 22:24:01 +00:00
LOG_FATAL(log, bare_stacktrace.str());
}
/// Write symbolized stack trace line by line for better grep-ability.
2020-05-23 22:24:01 +00:00
stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); });
#if defined(OS_LINUX)
/// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
String calculated_binary_hash = getHashOfLoadedBinaryHex();
if (daemon.stored_binary_hash.empty())
{
LOG_FATAL(log, "Calculated checksum of the binary: {}."
" There is no information about the reference checksum.", calculated_binary_hash);
}
else if (calculated_binary_hash == daemon.stored_binary_hash)
{
LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash);
}
else
{
LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond"
" to the reference checksum stored in the binary ({1})."
" It may indicate one of the following:"
" - the file was changed just after startup;"
" - the file is damaged on disk due to faulty hardware;"
" - the loaded executable is damaged in memory due to faulty hardware;"
" - the file was intentionally modified;"
" - logical error in code."
, calculated_binary_hash, daemon.stored_binary_hash);
}
#endif
2020-07-31 20:16:31 +00:00
/// Write crash to system.crash_log table if available.
2020-08-01 15:54:44 +00:00
if (collectCrashLog)
collectCrashLog(sig, thread_num, query_id, stack_trace);
2020-07-31 20:16:31 +00:00
/// Send crash report to developers (if configured)
if (sig != SanitizerTrap)
2020-07-31 20:16:31 +00:00
SentryWriter::onFault(sig, error_message, stack_trace);
2020-06-23 21:18:16 +00:00
2020-06-20 11:17:15 +00:00
/// When everything is done, we will try to send these error messages to client.
if (thread_ptr)
thread_ptr->onFatalError();
}
};
2020-05-12 03:31:53 +00:00
#if defined(SANITIZER)
extern "C" void __sanitizer_set_death_callback(void (*)());
static void sanitizerDeathCallback()
{
/// Also need to send data via pipe. Otherwise it may lead to deadlocks or failures in printing diagnostic info.
2020-05-12 03:31:53 +00:00
char buf[signal_pipe_buf_size];
DB::WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
2020-05-12 03:31:53 +00:00
const StackTrace stack_trace;
2020-05-12 03:31:53 +00:00
StringRef query_id = DB::CurrentThread::getQueryId();
query_id.size = std::min(query_id.size, max_query_id_size);
2020-05-12 03:31:53 +00:00
int sig = SignalListener::SanitizerTrap;
DB::writeBinary(sig, out);
DB::writePODBinary(stack_trace, out);
DB::writeBinary(UInt32(getThreadId()), out);
DB::writeStringBinary(query_id, out);
DB::writePODBinary(DB::current_thread, out);
out.next();
2020-05-12 03:31:53 +00:00
/// The time that is usually enough for separate thread to print info into log.
sleepForSeconds(10);
2020-05-12 03:31:53 +00:00
}
#endif
2018-01-17 18:06:39 +00:00
/** To use with std::set_terminate.
* Collects slightly more info than __gnu_cxx::__verbose_terminate_handler,
* and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log.
* Look at libstdc++-v3/libsupc++/vterminate.cc for example.
*/
[[noreturn]] static void terminate_handler()
{
static thread_local bool terminating = false;
if (terminating)
abort();
terminating = true;
std::string log_message;
if (std::current_exception())
log_message = "Terminate called for uncaught exception:\n" + DB::getCurrentExceptionMessage(true);
else
log_message = "Terminate called without an active exception";
2021-02-20 06:10:15 +00:00
/// POSIX.1 says that write(2)s of less than PIPE_BUF bytes must be atomic - man 7 pipe
/// And the buffer should not be too small because our exception messages can be large.
static constexpr size_t buf_size = PIPE_BUF;
if (log_message.size() > buf_size - 16)
log_message.resize(buf_size - 16);
char buf[buf_size];
DB::WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
DB::writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
2020-02-02 02:35:47 +00:00
DB::writeBinary(UInt32(getThreadId()), out);
DB::writeBinary(log_message, out);
out.next();
abort();
}
2018-02-14 20:41:46 +00:00
static std::string createDirectory(const std::string & file)
{
2021-05-22 18:24:13 +00:00
fs::path path = fs::path(file).parent_path();
if (path.empty())
2018-02-14 20:41:46 +00:00
return "";
2021-05-22 18:24:13 +00:00
fs::create_directories(path);
return path;
};
static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path)
{
try
{
2021-05-22 18:24:13 +00:00
fs::create_directories(path);
return true;
}
catch (...)
{
2020-05-23 22:24:01 +00:00
LOG_WARNING(logger, "{}: when creating {}, {}", __PRETTY_FUNCTION__, path, DB::getCurrentExceptionMessage(true));
}
return false;
}
void BaseDaemon::reloadConfiguration()
{
2018-01-17 18:06:39 +00:00
/** If the program is not run in daemon mode and 'config-file' is not specified,
* then we use config from 'config.xml' file in current directory,
* but will log to console (or use parameters --log-file, --errorlog-file from command line)
* instead of using files specified in config.xml.
* (It's convenient to log in console when you start server without any command line parameters.)
*/
2021-05-12 13:04:34 +00:00
config_path = config().getString("config-file", getDefaultConfigFileName());
DB::ConfigProcessor config_processor(config_path, false, true);
2021-05-22 18:24:13 +00:00
config_processor.setConfigPath(fs::path(config_path).parent_path());
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
if (last_configuration != nullptr)
config().removeConfiguration(last_configuration);
last_configuration = loaded_config.configuration.duplicate();
config().add(last_configuration, PRIO_DEFAULT, false);
}
2020-03-26 21:03:02 +00:00
BaseDaemon::BaseDaemon() = default;
2020-03-18 00:57:00 +00:00
BaseDaemon::~BaseDaemon()
{
writeSignalIDtoSignalPipe(SignalListener::StopThread);
signal_listener_thread.join();
Reset signal handlers just before closing signal pipe to avoid EBADF after stack trace: (gdb) bt 0 0x00007ffff7dda355 in raise () from /usr/lib/libc.so.6 1 0x00007ffff7dc3853 in abort () from /usr/lib/libc.so.6 2 0x0000000013a78180 in abort_message (format=format@entry=0x5781e10 "terminate_handler unexpectedly threw an exception") at abort_message.cpp:76 3 0x0000000013a899ff in std::__terminate (func=<optimized out>) at cxa_handlers.cpp:67 4 0x0000000013a771a1 in __cxxabiv1::call_terminate (native_exception=native_exception@entry=true, unwind_exception=unwind_exception@entry=0x7ffff70e9060) at cxa_personality.cpp:323 5 0x0000000013a77669 in __cxxabiv1::scan_eh_tab (results=..., actions=<optimized out>, native_exception=native_exception@entry=true, unwind_exception=0x7ffff70e9060, context=context@entry=0x7fffffffc3a0) at cxa_personality.cpp:887 6 0x0000000013a77bde in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=<optimized out>, exceptionClass=<optimized out>, unwind_exception=0x7ffff70e9060, context=0x7fffffffc3a0) at cxa_personality.cpp:969 7 0x0000000013a962d7 in unwind_phase1 (exception_object=0x7ffff70e9060, cursor=0x7fffffffc3a0, uc=0x7fffffffc2f0) at UnwindLevel1.c:98 8 _Unwind_RaiseException (exception_object=exception_object@entry=0x7ffff70e9060) at UnwindLevel1.c:363 9 0x0000000013a8929b in __cxxabiv1::__cxa_rethrow () at cxa_exception.cpp:607 10 0x000000000f170ad2 in DB::WriteBuffer::next (this=0x7fffffffc510) at WriteBuffer.h:52 11 writeSignalIDtoSignalPipe (sig=<optimized out>) at BaseDaemon.cpp:111 12 <signal handler called> 13 0x00007ffff7e9895b in munmap () from /usr/lib/libc.so.6 14 0x00007ffff7f7e255 in free_stacks () from /usr/lib/libpthread.so.0 15 0x00007ffff7f7e6aa in __deallocate_stack () from /usr/lib/libpthread.so.0 16 0x00007ffff7f80898 in __pthread_clockjoin_ex () from /usr/lib/libpthread.so.0 17 0x0000000013a759f7 in std::__1::__libcpp_thread_join (__t=0x7fff0aa2b090) at __threading_support:409 18 std::__1::thread::join (this=this@entry=0x7fff0aa2b090) at thread.cpp:56 19 0x000000000a4c9399 in ThreadPoolImpl<std::__1::thread>::finalize (this=this@entry=0x7ffff7053900) at list:355 20 0x000000000a4c9426 in ThreadPoolImpl<std::__1::thread>::~ThreadPoolImpl (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.cpp:172 21 0x000000000a4c8a9c in GlobalThreadPool::~GlobalThreadPool (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.h:132 22 std::__1::default_delete<GlobalThreadPool>::operator() (__ptr=0x7ffff7053900, this=<optimized out>) at memory:2363 23 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::reset (__p=0x0, this=<optimized out>) at memory:2618 24 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::~unique_ptr (this=<optimized out>, __in_chrg=<optimized out>) at memory:2572 25 0x00007ffff7ddcc57 in __run_exit_handlers () from /usr/lib/libc.so.6 26 0x00007ffff7ddcdfe in exit () from /usr/lib/libc.so.6 27 0x00007ffff7dc5009 in __libc_start_main () from /usr/lib/libc.so.6 28 0x000000000a48702e in _start () at new:340
2020-08-13 19:51:41 +00:00
/// Reset signals to SIG_DFL to avoid trying to write to the signal_pipe that will be closed after.
for (int sig : handled_signals)
{
signal(sig, SIG_DFL);
}
2020-03-18 00:57:00 +00:00
signal_pipe.close();
}
2019-02-02 12:26:07 +00:00
void BaseDaemon::terminate()
{
2018-08-20 23:16:50 +00:00
if (::raise(SIGTERM) != 0)
throw Poco::SystemException("cannot terminate process");
}
void BaseDaemon::kill()
{
dumpCoverageReportIfPossible();
2020-12-22 02:57:22 +00:00
pid_file.reset();
2021-01-07 18:36:38 +00:00
/// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL.
/// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker).
_exit(128 + SIGKILL);
}
2017-02-14 12:12:23 +00:00
std::string BaseDaemon::getDefaultCorePath() const
{
return "/opt/cores/";
}
2021-05-12 13:04:34 +00:00
std::string BaseDaemon::getDefaultConfigFileName() const
{
return "config.xml";
}
void BaseDaemon::closeFDs()
{
#if defined(OS_FREEBSD) || defined(OS_DARWIN)
2021-05-22 18:24:13 +00:00
fs::path proc_path{"/dev/fd"};
#else
2021-05-22 18:24:13 +00:00
fs::path proc_path{"/proc/self/fd"};
#endif
2021-05-22 18:24:13 +00:00
if (fs::is_directory(proc_path)) /// Hooray, proc exists
{
2021-05-27 07:35:36 +00:00
/// in /proc/self/fd directory filenames are numeric file descriptors.
/// Iterate directory separately from closing fds to avoid closing iterated directory fd.
std::vector<int> fds;
for (const auto & path : fs::directory_iterator(proc_path))
fds.push_back(DB::parse<int>(path.path().filename()));
for (const auto & fd : fds)
{
if (fd > 2 && fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1])
::close(fd);
}
}
else
{
2019-01-22 16:28:05 +00:00
int max_fd = -1;
#if defined(_SC_OPEN_MAX)
max_fd = sysconf(_SC_OPEN_MAX);
if (max_fd == -1)
#endif
max_fd = 256; /// bad fallback
2019-01-22 16:28:05 +00:00
for (int fd = 3; fd < max_fd; ++fd)
if (fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1])
::close(fd);
}
}
namespace
{
/// In debug version on Linux, increase oom score so that clickhouse is killed
/// first, instead of some service. Use a carefully chosen random score of 555:
/// the maximum is 1000, and chromium uses 300 for its tab processes. Ignore
/// whatever errors that occur, because it's just a debugging aid and we don't
/// care if it breaks.
#if defined(OS_LINUX) && !defined(NDEBUG)
void debugIncreaseOOMScore()
{
const std::string new_score = "555";
2019-07-25 20:17:17 +00:00
try
{
DB::WriteBufferFromFile buf("/proc/self/oom_score_adj");
buf.write(new_score.c_str(), new_score.size());
2021-02-13 10:15:36 +00:00
buf.close();
}
2019-07-25 20:17:17 +00:00
catch (const Poco::Exception & e)
{
2020-05-31 13:50:40 +00:00
LOG_WARNING(&Poco::Logger::root(), "Failed to adjust OOM score: '{}'.", e.displayText());
return;
}
2020-05-31 13:50:40 +00:00
LOG_INFO(&Poco::Logger::root(), "Set OOM score adjustment to {}", new_score);
}
#else
void debugIncreaseOOMScore() {}
#endif
}
void BaseDaemon::initialize(Application & self)
{
closeFDs();
ServerApplication::initialize(self);
/// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher!
argsToConfig(argv(), config(), PRIO_APPLICATION - 100);
2018-05-15 16:22:00 +00:00
bool is_daemon = config().getBool("application.runAsDaemon", false);
if (is_daemon)
{
2018-01-17 18:06:39 +00:00
/** When creating pid file and looking for config, will search for paths relative to the working path of the program when started.
*/
2021-05-22 18:24:13 +00:00
std::string path = fs::path(config().getString("application.path")).replace_filename("");
if (0 != chdir(path.c_str()))
throw Poco::Exception("Cannot change directory to " + path);
}
reloadConfiguration();
/// This must be done before creation of any files (including logs).
mode_t umask_num = 0027;
if (config().has("umask"))
{
std::string umask_str = config().getString("umask");
std::stringstream stream;
stream << umask_str;
stream >> std::oct >> umask_num;
}
umask(umask_num);
DB::ConfigProcessor(config_path).savePreprocessedConfig(loaded_config, "");
2018-01-17 18:06:39 +00:00
/// Write core dump on crash.
{
struct rlimit rlim;
if (getrlimit(RLIMIT_CORE, &rlim))
throw Poco::Exception("Cannot getrlimit");
2018-01-17 18:06:39 +00:00
/// 1 GiB by default. If more - it writes to disk too long.
rlim.rlim_cur = config().getUInt64("core_dump.size_limit", 1024 * 1024 * 1024);
if (rlim.rlim_cur && setrlimit(RLIMIT_CORE, &rlim))
{
2018-01-17 18:06:39 +00:00
/// It doesn't work under address/thread sanitizer. http://lists.llvm.org/pipermail/llvm-bugs/2013-April/027880.html
std::cerr << "Cannot set max size of core file to " + std::to_string(rlim.rlim_cur) << std::endl;
}
}
/// This must be done before any usage of DateLUT. In particular, before any logging.
if (config().has("timezone"))
{
const std::string config_timezone = config().getString("timezone");
if (0 != setenv("TZ", config_timezone.data(), 1))
throw Poco::Exception("Cannot setenv TZ variable");
tzset();
DateLUT::setDefaultTimezone(config_timezone);
}
std::string log_path = config().getString("logger.log", "");
if (!log_path.empty())
2021-05-22 18:24:13 +00:00
log_path = fs::path(log_path).replace_filename("");
/** Redirect stdout, stderr to separate files in the log directory (or in the specified file).
* Some libraries write to stderr in case of errors in debug mode,
* and this output makes sense even if the program is run in daemon mode.
* We have to do it before buildLoggers, for errors on logger initialization will be written to these files.
* If logger.stderr is specified then stderr will be forcibly redirected to that file.
*/
if ((!log_path.empty() && is_daemon) || config().has("logger.stderr"))
{
std::string stderr_path = config().getString("logger.stderr", log_path + "/stderr.log");
if (!freopen(stderr_path.c_str(), "a+", stderr))
throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path);
/// Disable buffering for stderr
setbuf(stderr, nullptr);
}
if ((!log_path.empty() && is_daemon) || config().has("logger.stdout"))
{
std::string stdout_path = config().getString("logger.stdout", log_path + "/stdout.log");
if (!freopen(stdout_path.c_str(), "a+", stdout))
throw Poco::OpenFileException("Cannot attach stdout to " + stdout_path);
}
/// Change path for logging.
if (!log_path.empty())
{
std::string path = createDirectory(log_path);
if (is_daemon
&& chdir(path.c_str()) != 0)
throw Poco::Exception("Cannot change directory to " + path);
}
else
{
if (is_daemon
&& chdir("/tmp") != 0)
throw Poco::Exception("Cannot change directory to /tmp");
}
2020-12-22 02:57:22 +00:00
/// sensitive data masking rules are not used here
2019-12-29 13:08:33 +00:00
buildLoggers(config(), logger(), self.commandName());
2020-12-22 02:57:22 +00:00
/// After initialized loggers but before initialized signal handling.
if (should_setup_watchdog)
setupWatchdog();
/// Create pid file.
if (config().has("pid"))
pid_file.emplace(config().getString("pid"), DB::StatusFile::write_pid);
if (is_daemon)
{
2018-01-17 18:06:39 +00:00
/** Change working directory to the directory to write core dumps.
* We have to do it after buildLoggers, because there is the case when config files was in current directory.
*/
std::string core_path = config().getString("core_path", "");
if (core_path.empty())
core_path = getDefaultCorePath();
tryCreateDirectories(&logger(), core_path);
2021-05-22 18:24:13 +00:00
if (!(fs::exists(core_path) && fs::is_directory(core_path)))
{
core_path = !log_path.empty() ? log_path : "/opt/";
tryCreateDirectories(&logger(), core_path);
}
if (0 != chdir(core_path.c_str()))
throw Poco::Exception("Cannot change directory to " + core_path);
}
initializeTerminationAndSignalProcessing();
logRevision();
debugIncreaseOOMScore();
for (const auto & key : DB::getMultipleKeysFromConfig(config(), "", "graphite"))
{
graphite_writers.emplace(key, std::make_unique<GraphiteWriter>(key));
}
}
2020-12-22 02:57:22 +00:00
static void addSignalHandler(const std::vector<int> & signals, signal_function handler, std::vector<int> * out_handled_signals)
{
2020-12-22 02:57:22 +00:00
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_sigaction = handler;
sa.sa_flags = SA_SIGINFO;
2020-05-10 22:17:28 +00:00
#if defined(OS_DARWIN)
2020-12-22 02:57:22 +00:00
sigemptyset(&sa.sa_mask);
for (auto signal : signals)
sigaddset(&sa.sa_mask, signal);
2020-05-10 22:17:28 +00:00
#else
2020-12-22 02:57:22 +00:00
if (sigemptyset(&sa.sa_mask))
throw Poco::Exception("Cannot set signal handler.");
2020-12-22 02:57:22 +00:00
for (auto signal : signals)
if (sigaddset(&sa.sa_mask, signal))
throw Poco::Exception("Cannot set signal handler.");
2020-05-10 22:17:28 +00:00
#endif
2020-12-22 02:57:22 +00:00
for (auto signal : signals)
if (sigaction(signal, &sa, nullptr))
throw Poco::Exception("Cannot set signal handler.");
Reset signal handlers just before closing signal pipe to avoid EBADF after stack trace: (gdb) bt 0 0x00007ffff7dda355 in raise () from /usr/lib/libc.so.6 1 0x00007ffff7dc3853 in abort () from /usr/lib/libc.so.6 2 0x0000000013a78180 in abort_message (format=format@entry=0x5781e10 "terminate_handler unexpectedly threw an exception") at abort_message.cpp:76 3 0x0000000013a899ff in std::__terminate (func=<optimized out>) at cxa_handlers.cpp:67 4 0x0000000013a771a1 in __cxxabiv1::call_terminate (native_exception=native_exception@entry=true, unwind_exception=unwind_exception@entry=0x7ffff70e9060) at cxa_personality.cpp:323 5 0x0000000013a77669 in __cxxabiv1::scan_eh_tab (results=..., actions=<optimized out>, native_exception=native_exception@entry=true, unwind_exception=0x7ffff70e9060, context=context@entry=0x7fffffffc3a0) at cxa_personality.cpp:887 6 0x0000000013a77bde in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=<optimized out>, exceptionClass=<optimized out>, unwind_exception=0x7ffff70e9060, context=0x7fffffffc3a0) at cxa_personality.cpp:969 7 0x0000000013a962d7 in unwind_phase1 (exception_object=0x7ffff70e9060, cursor=0x7fffffffc3a0, uc=0x7fffffffc2f0) at UnwindLevel1.c:98 8 _Unwind_RaiseException (exception_object=exception_object@entry=0x7ffff70e9060) at UnwindLevel1.c:363 9 0x0000000013a8929b in __cxxabiv1::__cxa_rethrow () at cxa_exception.cpp:607 10 0x000000000f170ad2 in DB::WriteBuffer::next (this=0x7fffffffc510) at WriteBuffer.h:52 11 writeSignalIDtoSignalPipe (sig=<optimized out>) at BaseDaemon.cpp:111 12 <signal handler called> 13 0x00007ffff7e9895b in munmap () from /usr/lib/libc.so.6 14 0x00007ffff7f7e255 in free_stacks () from /usr/lib/libpthread.so.0 15 0x00007ffff7f7e6aa in __deallocate_stack () from /usr/lib/libpthread.so.0 16 0x00007ffff7f80898 in __pthread_clockjoin_ex () from /usr/lib/libpthread.so.0 17 0x0000000013a759f7 in std::__1::__libcpp_thread_join (__t=0x7fff0aa2b090) at __threading_support:409 18 std::__1::thread::join (this=this@entry=0x7fff0aa2b090) at thread.cpp:56 19 0x000000000a4c9399 in ThreadPoolImpl<std::__1::thread>::finalize (this=this@entry=0x7ffff7053900) at list:355 20 0x000000000a4c9426 in ThreadPoolImpl<std::__1::thread>::~ThreadPoolImpl (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.cpp:172 21 0x000000000a4c8a9c in GlobalThreadPool::~GlobalThreadPool (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.h:132 22 std::__1::default_delete<GlobalThreadPool>::operator() (__ptr=0x7ffff7053900, this=<optimized out>) at memory:2363 23 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::reset (__p=0x0, this=<optimized out>) at memory:2618 24 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::~unique_ptr (this=<optimized out>, __in_chrg=<optimized out>) at memory:2572 25 0x00007ffff7ddcc57 in __run_exit_handlers () from /usr/lib/libc.so.6 26 0x00007ffff7ddcdfe in exit () from /usr/lib/libc.so.6 27 0x00007ffff7dc5009 in __libc_start_main () from /usr/lib/libc.so.6 28 0x000000000a48702e in _start () at new:340
2020-08-13 19:51:41 +00:00
2020-12-22 02:57:22 +00:00
if (out_handled_signals)
std::copy(signals.begin(), signals.end(), std::back_inserter(*out_handled_signals));
};
static void blockSignals(const std::vector<int> & signals)
{
sigset_t sig_set;
2020-05-10 22:17:28 +00:00
#if defined(OS_DARWIN)
2020-12-22 17:13:18 +00:00
sigemptyset(&sig_set);
for (auto signal : signals)
sigaddset(&sig_set, signal);
2020-05-10 22:17:28 +00:00
#else
2020-12-22 02:57:22 +00:00
if (sigemptyset(&sig_set))
throw Poco::Exception("Cannot block signal.");
2020-12-22 02:57:22 +00:00
for (auto signal : signals)
if (sigaddset(&sig_set, signal))
throw Poco::Exception("Cannot block signal.");
2020-05-10 22:17:28 +00:00
#endif
2020-12-22 02:57:22 +00:00
if (pthread_sigmask(SIG_BLOCK, &sig_set, nullptr))
throw Poco::Exception("Cannot block signal.");
};
Reset signal handlers just before closing signal pipe to avoid EBADF after stack trace: (gdb) bt 0 0x00007ffff7dda355 in raise () from /usr/lib/libc.so.6 1 0x00007ffff7dc3853 in abort () from /usr/lib/libc.so.6 2 0x0000000013a78180 in abort_message (format=format@entry=0x5781e10 "terminate_handler unexpectedly threw an exception") at abort_message.cpp:76 3 0x0000000013a899ff in std::__terminate (func=<optimized out>) at cxa_handlers.cpp:67 4 0x0000000013a771a1 in __cxxabiv1::call_terminate (native_exception=native_exception@entry=true, unwind_exception=unwind_exception@entry=0x7ffff70e9060) at cxa_personality.cpp:323 5 0x0000000013a77669 in __cxxabiv1::scan_eh_tab (results=..., actions=<optimized out>, native_exception=native_exception@entry=true, unwind_exception=0x7ffff70e9060, context=context@entry=0x7fffffffc3a0) at cxa_personality.cpp:887 6 0x0000000013a77bde in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=<optimized out>, exceptionClass=<optimized out>, unwind_exception=0x7ffff70e9060, context=0x7fffffffc3a0) at cxa_personality.cpp:969 7 0x0000000013a962d7 in unwind_phase1 (exception_object=0x7ffff70e9060, cursor=0x7fffffffc3a0, uc=0x7fffffffc2f0) at UnwindLevel1.c:98 8 _Unwind_RaiseException (exception_object=exception_object@entry=0x7ffff70e9060) at UnwindLevel1.c:363 9 0x0000000013a8929b in __cxxabiv1::__cxa_rethrow () at cxa_exception.cpp:607 10 0x000000000f170ad2 in DB::WriteBuffer::next (this=0x7fffffffc510) at WriteBuffer.h:52 11 writeSignalIDtoSignalPipe (sig=<optimized out>) at BaseDaemon.cpp:111 12 <signal handler called> 13 0x00007ffff7e9895b in munmap () from /usr/lib/libc.so.6 14 0x00007ffff7f7e255 in free_stacks () from /usr/lib/libpthread.so.0 15 0x00007ffff7f7e6aa in __deallocate_stack () from /usr/lib/libpthread.so.0 16 0x00007ffff7f80898 in __pthread_clockjoin_ex () from /usr/lib/libpthread.so.0 17 0x0000000013a759f7 in std::__1::__libcpp_thread_join (__t=0x7fff0aa2b090) at __threading_support:409 18 std::__1::thread::join (this=this@entry=0x7fff0aa2b090) at thread.cpp:56 19 0x000000000a4c9399 in ThreadPoolImpl<std::__1::thread>::finalize (this=this@entry=0x7ffff7053900) at list:355 20 0x000000000a4c9426 in ThreadPoolImpl<std::__1::thread>::~ThreadPoolImpl (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.cpp:172 21 0x000000000a4c8a9c in GlobalThreadPool::~GlobalThreadPool (this=0x7ffff7053900, __in_chrg=<optimized out>) at ThreadPool.h:132 22 std::__1::default_delete<GlobalThreadPool>::operator() (__ptr=0x7ffff7053900, this=<optimized out>) at memory:2363 23 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::reset (__p=0x0, this=<optimized out>) at memory:2618 24 std::__1::unique_ptr<GlobalThreadPool, std::__1::default_delete<GlobalThreadPool> >::~unique_ptr (this=<optimized out>, __in_chrg=<optimized out>) at memory:2572 25 0x00007ffff7ddcc57 in __run_exit_handlers () from /usr/lib/libc.so.6 26 0x00007ffff7ddcdfe in exit () from /usr/lib/libc.so.6 27 0x00007ffff7dc5009 in __libc_start_main () from /usr/lib/libc.so.6 28 0x000000000a48702e in _start () at new:340
2020-08-13 19:51:41 +00:00
2020-12-22 02:57:22 +00:00
void BaseDaemon::initializeTerminationAndSignalProcessing()
{
SentryWriter::initialize(config());
std::set_terminate(terminate_handler);
2020-12-22 02:57:22 +00:00
/// We want to avoid SIGPIPE when working with sockets and pipes, and just handle return value/errno instead.
blockSignals({SIGPIPE});
2020-12-22 02:57:22 +00:00
/// Setup signal handlers.
2019-12-02 17:29:19 +00:00
/// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
2021-02-12 21:29:47 +00:00
addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, &handled_signals);
2020-12-22 02:57:22 +00:00
addSignalHandler({SIGHUP, SIGUSR1}, closeLogsSignalHandler, &handled_signals);
addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, &handled_signals);
2020-05-12 03:31:53 +00:00
#if defined(SANITIZER)
__sanitizer_set_death_callback(sanitizerDeathCallback);
#endif
2018-01-17 18:06:39 +00:00
/// Set up Poco ErrorHandler for Poco Threads.
static KillingErrorHandler killing_error_handler;
Poco::ErrorHandler::set(&killing_error_handler);
2020-11-27 10:12:11 +00:00
signal_pipe.setNonBlockingWrite();
signal_pipe.tryIncreaseSize(1 << 20);
2020-03-08 21:04:10 +00:00
signal_listener = std::make_unique<SignalListener>(*this);
signal_listener_thread.start(*signal_listener);
2020-06-20 09:07:05 +00:00
2020-06-20 09:17:31 +00:00
#if defined(__ELF__) && !defined(__FreeBSD__)
2020-11-30 14:30:55 +00:00
String build_id_hex = DB::SymbolIndex::instance()->getBuildIDHex();
2020-06-20 09:07:05 +00:00
if (build_id_hex.empty())
build_id_info = "no build id";
else
build_id_info = "build id: " + build_id_hex;
2020-06-20 09:17:31 +00:00
#else
build_id_info = "no build id";
#endif
#if defined(__linux__)
std::string executable_path = getExecutablePath();
if (!executable_path.empty())
stored_binary_hash = DB::Elf(executable_path).getBinaryHash();
#endif
}
2016-02-10 07:54:22 +00:00
void BaseDaemon::logRevision() const
{
2020-05-30 21:57:37 +00:00
Poco::Logger::root().information("Starting " + std::string{VERSION_FULL}
2020-09-17 12:15:05 +00:00
+ " with revision " + std::to_string(ClickHouseRevision::getVersionRevision())
2020-06-20 09:13:11 +00:00
+ ", " + build_id_info
2019-09-26 12:08:24 +00:00
+ ", PID " + std::to_string(getpid()));
}
2020-05-09 23:22:51 +00:00
void BaseDaemon::defineOptions(Poco::Util::OptionSet & new_options)
{
2020-05-09 23:22:51 +00:00
new_options.addOption(
2018-01-17 16:39:56 +00:00
Poco::Util::Option("config-file", "C", "load configuration from a given file")
.required(false)
.repeatable(false)
.argument("<file>")
.binding("config-file"));
2020-05-09 23:22:51 +00:00
new_options.addOption(
2018-01-17 16:39:56 +00:00
Poco::Util::Option("log-file", "L", "use given log file")
.required(false)
.repeatable(false)
.argument("<file>")
.binding("logger.log"));
2020-05-09 23:22:51 +00:00
new_options.addOption(
2018-01-17 16:39:56 +00:00
Poco::Util::Option("errorlog-file", "E", "use given log file for errors only")
.required(false)
.repeatable(false)
.argument("<file>")
.binding("logger.errorlog"));
2020-05-09 23:22:51 +00:00
new_options.addOption(
2018-01-17 16:39:56 +00:00
Poco::Util::Option("pid-file", "P", "use given pidfile")
.required(false)
.repeatable(false)
.argument("<file>")
.binding("pid"));
2020-05-09 23:22:51 +00:00
Poco::Util::ServerApplication::defineOptions(new_options);
}
void BaseDaemon::handleSignal(int signal_id)
{
if (signal_id == SIGINT ||
signal_id == SIGQUIT ||
signal_id == SIGTERM)
{
std::unique_lock<std::mutex> lock(signal_handler_mutex);
{
++terminate_signals_counter;
sigint_signals_counter += signal_id == SIGINT;
signal_event.notify_all();
}
onInterruptSignals(signal_id);
}
else
2021-01-08 05:09:30 +00:00
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
}
void BaseDaemon::onInterruptSignals(int signal_id)
{
is_cancelled = true;
2021-01-08 05:09:30 +00:00
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));
if (sigint_signals_counter >= 2)
{
2020-05-23 22:24:01 +00:00
LOG_INFO(&logger(), "Received second signal Interrupt. Immediately terminate.");
2020-12-22 02:57:22 +00:00
call_default_signal_handler(signal_id);
/// If the above did not help.
_exit(128 + signal_id);
}
}
void BaseDaemon::waitForTerminationRequest()
{
/// NOTE: as we already process signals via pipe, we don't have to block them with sigprocmask in threads
std::unique_lock<std::mutex> lock(signal_handler_mutex);
signal_event.wait(lock, [this](){ return terminate_signals_counter > 0; });
}
2020-12-22 02:57:22 +00:00
void BaseDaemon::shouldSetupWatchdog(char * argv0_)
{
should_setup_watchdog = true;
argv0 = argv0_;
}
void BaseDaemon::setupWatchdog()
{
2020-12-22 07:38:39 +00:00
/// Initialize in advance to avoid double initialization in forked processes.
DateLUT::instance();
2020-12-22 02:57:22 +00:00
std::string original_process_name;
if (argv0)
original_process_name = argv0;
while (true)
{
2020-12-22 03:17:31 +00:00
static pid_t pid = -1;
pid = fork();
2020-12-22 02:57:22 +00:00
if (-1 == pid)
throw Poco::Exception("Cannot fork");
if (0 == pid)
{
logger().information("Forked a child process to watch");
2020-12-22 03:17:31 +00:00
#if defined(__linux__)
if (0 != prctl(PR_SET_PDEATHSIG, SIGKILL))
logger().warning("Cannot do prctl to ask termination with parent.");
#endif
2020-12-22 02:57:22 +00:00
return;
}
/// Change short thread name and process name.
setThreadName("clckhouse-watch"); /// 15 characters
if (argv0)
{
const char * new_process_name = "clickhouse-watchdog";
memset(argv0, 0, original_process_name.size());
memcpy(argv0, new_process_name, std::min(strlen(new_process_name), original_process_name.size()));
}
logger().information(fmt::format("Will watch for the process with pid {}", pid));
2020-12-22 03:17:31 +00:00
/// Forward signals to the child process.
2020-12-22 02:57:22 +00:00
addSignalHandler(
{SIGHUP, SIGUSR1, SIGINT, SIGQUIT, SIGTERM},
2020-12-22 03:17:31 +00:00
[](int sig, siginfo_t *, void *)
{
2020-12-22 06:45:08 +00:00
/// Forward all signals except INT as it can be send by terminal to the process group when user press Ctrl+C,
/// and we process double delivery of this signal as immediate termination.
2020-12-22 03:17:31 +00:00
if (sig == SIGINT)
return;
const char * error_message = "Cannot forward signal to the child process.\n";
if (0 != ::kill(pid, sig))
2020-12-22 06:45:08 +00:00
{
auto res = write(STDERR_FILENO, error_message, strlen(error_message));
(void)res;
}
2020-12-22 03:17:31 +00:00
},
nullptr);
2020-12-22 02:57:22 +00:00
int status = 0;
do
{
if (-1 != waitpid(pid, &status, WUNTRACED | WCONTINUED) || errno == ECHILD)
{
if (WIFSTOPPED(status))
logger().warning(fmt::format("Child process was stopped by signal {}.", WSTOPSIG(status)));
else if (WIFCONTINUED(status))
logger().warning(fmt::format("Child process was continued."));
else
break;
}
else if (errno != EINTR)
throw Poco::Exception("Cannot waitpid, errno: " + std::string(strerror(errno)));
} while (true);
if (errno == ECHILD)
{
logger().information("Child process no longer exists.");
_exit(WEXITSTATUS(status));
2020-12-22 02:57:22 +00:00
}
if (WIFEXITED(status))
{
logger().information(fmt::format("Child process exited normally with code {}.", WEXITSTATUS(status)));
2020-12-29 17:36:10 +00:00
_exit(WEXITSTATUS(status));
2020-12-22 02:57:22 +00:00
}
if (WIFSIGNALED(status))
{
int sig = WTERMSIG(status);
if (sig == SIGKILL)
{
logger().fatal(fmt::format("Child process was terminated by signal {} (KILL)."
" If it is not done by 'forcestop' command or manually,"
" the possible cause is OOM Killer (see 'dmesg' and look at the '/var/log/kern.log' for the details).", sig));
}
else
{
logger().fatal(fmt::format("Child process was terminated by signal {}.", sig));
if (sig == SIGINT || sig == SIGTERM || sig == SIGQUIT)
2020-12-29 17:36:10 +00:00
_exit(128 + sig);
2020-12-22 02:57:22 +00:00
}
}
else
{
logger().fatal("Child process was not exited normally by unknown reason.");
}
/// Automatic restart is not enabled but you can play with it.
#if 1
_exit(WEXITSTATUS(status));
2020-12-22 02:57:22 +00:00
#else
logger().information("Will restart.");
if (argv0)
memcpy(argv0, original_process_name.c_str(), original_process_name.size());
#endif
}
}
String BaseDaemon::getStoredBinaryHash() const
{
return stored_binary_hash;
}