ClickHouse/src/Interpreters/SystemLog.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

163 lines
5.7 KiB
C++
Raw Normal View History

2016-10-27 17:48:12 +00:00
#pragma once
2022-01-25 09:58:11 +00:00
#include <Common/SystemLogBase.h>
2016-10-27 17:48:12 +00:00
2022-01-25 09:58:11 +00:00
#include <Interpreters/StorageID.h>
2016-10-27 17:48:12 +00:00
namespace DB
{
/** Allow to store structured log in system table.
*
* Logging is asynchronous. Data is put into queue from where it will be read by separate thread.
* That thread inserts log into a table with no more than specified periodicity.
*/
/** Structure of log, template parameter.
* Structure could change on server version update.
* If on first write, existing table has different structure,
* then it get renamed (put aside) and new table is created.
*/
/* Example:
struct LogElement
{
/// default constructor must be available
/// fields
static std::string name();
static NamesAndTypesList getNamesAndTypes();
static NamesAndAliases getNamesAndAliases();
static const char * getCustomColumnList();
void appendToBlock(MutableColumns & columns) const;
2016-10-27 17:48:12 +00:00
};
*/
class QueryLog;
class QueryThreadLog;
class PartLog;
class TextLog;
class TraceLog;
2020-07-09 04:15:45 +00:00
class CrashLog;
class MetricLog;
2020-06-10 19:17:30 +00:00
class AsynchronousMetricLog;
2020-08-20 20:59:40 +00:00
class OpenTelemetrySpanLog;
2021-06-18 13:44:08 +00:00
class QueryViewsLog;
2021-07-09 14:05:35 +00:00
class ZooKeeperLog;
class SessionLog;
2022-01-14 14:03:00 +00:00
class TransactionsInfoLog;
class ProcessorsProfileLog;
2022-04-30 05:00:40 +00:00
class FilesystemCacheLog;
2023-02-07 17:50:31 +00:00
class FilesystemReadPrefetchesLog;
class AsynchronousInsertLog;
2020-04-13 01:33:05 +00:00
/// System logs should be destroyed in destructor of the last Context and before tables,
/// because SystemLog destruction makes insert query while flushing data into underlying tables
struct SystemLogs
{
SystemLogs(ContextPtr global_context, const Poco::Util::AbstractConfiguration & config);
~SystemLogs();
2019-06-21 17:25:47 +00:00
void shutdown();
2019-03-21 19:22:38 +00:00
std::shared_ptr<QueryLog> query_log; /// Used to log queries.
std::shared_ptr<QueryThreadLog> query_thread_log; /// Used to log query threads.
std::shared_ptr<PartLog> part_log; /// Used to log operations with parts
std::shared_ptr<TraceLog> trace_log; /// Used to log traces from query profiler
2020-07-09 04:15:45 +00:00
std::shared_ptr<CrashLog> crash_log; /// Used to log server crashes.
2019-08-04 15:51:04 +00:00
std::shared_ptr<TextLog> text_log; /// Used to log all text messages.
2019-08-13 14:31:46 +00:00
std::shared_ptr<MetricLog> metric_log; /// Used to log all metrics.
2023-02-07 17:50:31 +00:00
std::shared_ptr<FilesystemCacheLog> filesystem_cache_log;
std::shared_ptr<FilesystemReadPrefetchesLog> filesystem_read_prefetches_log;
2020-06-10 19:17:30 +00:00
/// Metrics from system.asynchronous_metrics.
std::shared_ptr<AsynchronousMetricLog> asynchronous_metric_log;
2020-10-22 16:47:20 +00:00
/// OpenTelemetry trace spans.
std::shared_ptr<OpenTelemetrySpanLog> opentelemetry_span_log;
2021-06-18 13:44:08 +00:00
/// Used to log queries of materialized and live views
std::shared_ptr<QueryViewsLog> query_views_log;
2021-07-09 14:05:35 +00:00
/// Used to log all actions of ZooKeeper client
std::shared_ptr<ZooKeeperLog> zookeeper_log;
/// Login, LogOut and Login failure events
std::shared_ptr<SessionLog> session_log;
2022-01-14 14:03:00 +00:00
/// Events related to transactions
std::shared_ptr<TransactionsInfoLog> transactions_info_log;
/// Used to log processors profiling
std::shared_ptr<ProcessorsProfileLog> processors_profile_log;
std::shared_ptr<AsynchronousInsertLog> asynchronous_insert_log;
2020-04-13 01:33:05 +00:00
std::vector<ISystemLog *> logs;
};
2016-10-27 17:48:12 +00:00
template <typename LogElement>
2023-07-18 10:06:44 +00:00
class SystemLog : public ISystemLog, private boost::noncopyable, WithContext
2016-10-27 17:48:12 +00:00
{
public:
using Self = SystemLog;
2016-10-27 17:48:12 +00:00
/** Parameter: table name where to write log.
* If table is not exists, then it get created with specified engine.
* If it already exists, then its structure is checked to be compatible with structure of log record.
* If it is compatible, then existing table will be used.
* If not - then existing table will be renamed to same name but with suffix '_N' at end,
* where N - is a minimal number from 1, for that table with corresponding name doesn't exist yet;
* and new table get created - as if previous table was not exist.
*/
SystemLog(
ContextPtr context_,
2016-10-27 17:48:12 +00:00
const String & database_name_,
const String & table_name_,
const String & storage_def_,
2023-07-12 15:48:09 +00:00
size_t flush_interval_milliseconds_,
2023-07-17 05:36:24 +00:00
std::shared_ptr<SystemLogQueue<LogElement>> queue_ = nullptr);
2023-07-18 10:06:44 +00:00
void startup() override;
/** Append a record into log.
* Writing to table will be done asynchronously and in case of failure, record could be lost.
*/
void add(const LogElement & element);
void shutdown() override;
2020-04-13 01:33:05 +00:00
2023-07-18 10:06:44 +00:00
String getName() const override { return LogElement::name(); }
static const char * getDefaultOrderBy() { return "event_date, event_time"; }
/// Flush data in the buffer to disk. Block the thread until the data is stored on disk.
void flush(bool force) override;
/// Non-blocking flush data in the buffer to disk.
void notifyFlush(bool force);
void stopFlushThread() override;
2023-07-18 14:33:26 +00:00
protected:
2023-07-18 10:06:44 +00:00
Poco::Logger * log;
2023-07-18 14:33:26 +00:00
2023-07-18 10:06:44 +00:00
using ISystemLog::is_shutdown;
using ISystemLog::saving_thread;
private:
2022-01-25 09:58:11 +00:00
2023-07-18 10:06:44 +00:00
/* Saving thread data */
2020-02-17 19:28:25 +00:00
const StorageID table_id;
const String storage_def;
String create_query;
String old_create_query;
bool is_prepared = false;
2023-07-18 10:06:44 +00:00
std::shared_ptr<SystemLogQueue<LogElement>> queue;
/** Creates new table if it does not exist.
* Renames old table if its structure is not suitable.
* This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
*/
2020-04-19 23:11:41 +00:00
void prepareTable() override;
2019-03-21 19:22:38 +00:00
2022-01-25 09:58:11 +00:00
void savingThreadFunction() override;
/// flushImpl can be executed only in saving_thread.
void flushImpl(const std::vector<LogElement> & to_flush, uint64_t to_flush_end);
2022-01-25 09:58:11 +00:00
ASTPtr getCreateTableQuery();
2016-10-27 17:48:12 +00:00
};
}