From c6667ff88848a0c3bf3ee75fc496579556fbfeb7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 22 Jul 2019 16:54:08 +0300 Subject: [PATCH 01/26] wip --- dbms/src/Interpreters/Context.cpp | 10 +++ dbms/src/Interpreters/Context.h | 2 + dbms/src/Interpreters/SystemLog.cpp | 5 +- dbms/src/Interpreters/SystemLog.h | 2 + dbms/src/Interpreters/TextLog.cpp | 69 +++++++++++++++++++++ dbms/src/Interpreters/TextLog.h | 41 ++++++++++++ libs/libloggers/loggers/OwnSplitChannel.cpp | 30 ++++++++- 7 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 dbms/src/Interpreters/TextLog.cpp create mode 100644 dbms/src/Interpreters/TextLog.h diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index bf779ada6b4..b5866ed3bef 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1663,6 +1663,16 @@ std::shared_ptr Context::getPartLog(const String & part_database) return shared->system_logs->part_log; } +std::shared_ptr Context::getTextLog() +{ + auto lock = getLock(); + + if (!shared->system_logs || !shared->system_logs->text_log) + return {}; + + return shared->system_logs->text_log; +} + CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 44547f97ef2..bc471ae39cc 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -62,6 +62,7 @@ class Clusters; class QueryLog; class QueryThreadLog; class PartLog; +class TextLog; struct MergeTreeSettings; class IDatabase; class DDLGuard; @@ -423,6 +424,7 @@ public: /// Nullptr if the query log is not ready for this moment. std::shared_ptr getQueryLog(); std::shared_ptr getQueryThreadLog(); + std::shared_ptr getTextLog(); /// Returns an object used to log opertaions with parts if it possible. /// Provide table name to make required cheks. diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index f46b348db7a..624c6a8ed75 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -2,7 +2,7 @@ #include #include #include - +#include #include @@ -44,6 +44,7 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi query_log = createSystemLog(global_context, "system", "query_log", config, "query_log"); query_thread_log = createSystemLog(global_context, "system", "query_thread_log", config, "query_thread_log"); part_log = createSystemLog(global_context, "system", "part_log", config, "part_log"); + text_log = createSystemLog(global_context, "system", "text_log", config, "text_log"); part_log_database = config.getString("part_log.database", "system"); } @@ -63,6 +64,8 @@ void SystemLogs::shutdown() query_thread_log->shutdown(); if (part_log) part_log->shutdown(); + if (text_log) + text_log->shutdown(); } } diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 48dbde5a38b..83f4d6bbc12 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -59,6 +59,7 @@ class Context; class QueryLog; class QueryThreadLog; class PartLog; +class TextLog; /// System logs should be destroyed in destructor of the last Context and before tables, @@ -73,6 +74,7 @@ struct SystemLogs std::shared_ptr query_log; /// Used to log queries. std::shared_ptr query_thread_log; /// Used to log query threads. std::shared_ptr part_log; /// Used to log operations with parts + std::shared_ptr text_log; String part_log_database; }; diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp new file mode 100644 index 00000000000..a3da1100ec4 --- /dev/null +++ b/dbms/src/Interpreters/TextLog.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB { + +Block TextLogElement::createBlock() { + return + { + {std::make_shared(), "event_date"}, + {std::make_shared(), "event_time"}, + {std::make_shared(), "microseconds"}, + + {std::make_shared(std::make_shared()), "thread_name"}, + {std::make_shared(), "thread_number"}, + {std::make_shared(), "os_thread_id"}, + + {std::make_shared(), "level"}, + + {std::make_shared(), "query_id"}, + {std::make_shared(), "logger_name"}, + {std::make_shared(), "message"}, + + {std::make_shared(std::make_shared()), "source_file"}, + {std::make_shared(), "source_line"} + }; +} + +void TextLogElement::appendToBlock(Block & block) const +{ + MutableColumns columns = block.mutateColumns(); + + size_t i = 0; + + columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + + columns[i++]->insert(event_time); + columns[i++]->insert(microseconds); + + // Thread info + columns[i++]->insertData(thread_name.data(), thread_name.size()); + columns[i++]->insert(thread_number); + columns[i++]->insert(os_thread_id); + + columns[i++]->insert(UInt8(level)); + + columns[i++]->insert(query_id); + columns[i++]->insert(logger_name); + columns[i++]->insert(message); + + columns[i++]->insert(source_file); + columns[i++]->insert(source_line); +} + +} + diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h new file mode 100644 index 00000000000..1caca571fe1 --- /dev/null +++ b/dbms/src/Interpreters/TextLog.h @@ -0,0 +1,41 @@ +#pragma once +#include + +namespace DB { + +using Poco::Message; + +struct TextLogElement +{ + time_t event_date{}; + time_t event_time{}; + + UInt32 microseconds; + + String thread_name; + UInt32 os_thread_id; + UInt32 thread_number; + + Message::Priority level; + + String query_id; + + String logger_name; + String message; + + String source_file; + UInt64 source_line; + + static std::string name() { return "TextLog"; } + static Block createBlock(); + void appendToBlock(Block & block) const; +}; + +class TextLog : public SystemLog +{ + using SystemLog::SystemLog; +}; + + +} //namespace DB + diff --git a/libs/libloggers/loggers/OwnSplitChannel.cpp b/libs/libloggers/loggers/OwnSplitChannel.cpp index 0779f6477c3..b96296ee364 100644 --- a/libs/libloggers/loggers/OwnSplitChannel.cpp +++ b/libs/libloggers/loggers/OwnSplitChannel.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,34 @@ void OwnSplitChannel::log(const Poco::Message & msg) logs_queue->emplace(std::move(columns)); } - /// TODO: Also log to system.internal_text_log table + /// Also log to system.internal_text_log table + + if (CurrentThread::getGroup() != nullptr && + CurrentThread::getGroup()->global_context != nullptr && + CurrentThread::getGroup()->global_context->getTextLog() != nullptr) { + + const auto text_log = CurrentThread::getGroup()->global_context->getTextLog(); + TextLogElement elem; + + elem.event_time = msg_ext.time_seconds; + elem.microseconds = msg_ext.time_microseconds; + + elem.thread_name = getThreadName(); + elem.thread_number = msg_ext.thread_number; + elem.os_thread_id = msg.getOsTid(); + elem.message = msg.getText(); + + elem.level = msg.getPriority(); + + if (msg.getSourceFile() != nullptr) { + elem.source_file = msg.getSourceFile(); + } + + elem.source_line = msg.getSourceLine(); + + text_log->add(elem); + } + } void OwnSplitChannel::addChannel(Poco::AutoPtr channel) From 8289472b9821cc3e813036febb72bb8af8b67765 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 22 Jul 2019 18:09:33 +0300 Subject: [PATCH 02/26] style improvements --- dbms/src/Interpreters/TextLog.cpp | 32 +++++++++++++++++---- dbms/src/Interpreters/TextLog.h | 8 +++--- libs/libloggers/loggers/OwnSplitChannel.cpp | 6 ++-- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp index a3da1100ec4..d11f6246319 100644 --- a/dbms/src/Interpreters/TextLog.cpp +++ b/dbms/src/Interpreters/TextLog.cpp @@ -15,9 +15,27 @@ #include #include -namespace DB { +namespace DB +{ + +template <> struct NearestFieldTypeImpl { using Type = UInt64; }; + +Block TextLogElement::createBlock() +{ + + auto priority_datatype = std::make_shared( + DataTypeEnum8::Values + { + {"FATAL", static_cast(Message::PRIO_FATAL)}, + {"CRITICAL", static_cast(Message::PRIO_CRITICAL)}, + {"ERROR", static_cast(Message::PRIO_ERROR)}, + {"WARNING", static_cast(Message::PRIO_WARNING)}, + {"NOTICE", static_cast(Message::PRIO_NOTICE)}, + {"INFORMATION", static_cast(Message::PRIO_INFORMATION)}, + {"DEBUG", static_cast(Message::PRIO_DEBUG)}, + {"TRACE", static_cast(Message::PRIO_TRACE)} + }); -Block TextLogElement::createBlock() { return { {std::make_shared(), "event_date"}, @@ -28,12 +46,14 @@ Block TextLogElement::createBlock() { {std::make_shared(), "thread_number"}, {std::make_shared(), "os_thread_id"}, - {std::make_shared(), "level"}, + {std::move(priority_datatype), "level"}, {std::make_shared(), "query_id"}, {std::make_shared(), "logger_name"}, {std::make_shared(), "message"}, + {std::make_shared(), "revision"}, + {std::make_shared(std::make_shared()), "source_file"}, {std::make_shared(), "source_line"} }; @@ -46,7 +66,6 @@ void TextLogElement::appendToBlock(Block & block) const size_t i = 0; columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); - columns[i++]->insert(event_time); columns[i++]->insert(microseconds); @@ -55,15 +74,16 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(thread_number); columns[i++]->insert(os_thread_id); - columns[i++]->insert(UInt8(level)); + columns[i++]->insert(level); columns[i++]->insert(query_id); columns[i++]->insert(logger_name); columns[i++]->insert(message); + columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(source_file); columns[i++]->insert(source_line); } } - diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h index 1caca571fe1..a8d76896b01 100644 --- a/dbms/src/Interpreters/TextLog.h +++ b/dbms/src/Interpreters/TextLog.h @@ -1,7 +1,8 @@ #pragma once #include -namespace DB { +namespace DB +{ using Poco::Message; @@ -16,7 +17,7 @@ struct TextLogElement UInt32 os_thread_id; UInt32 thread_number; - Message::Priority level; + Message::Priority level = Message::PRIO_TRACE; String query_id; @@ -37,5 +38,4 @@ class TextLog : public SystemLog }; -} //namespace DB - +} diff --git a/libs/libloggers/loggers/OwnSplitChannel.cpp b/libs/libloggers/loggers/OwnSplitChannel.cpp index b96296ee364..c9f8f5e8bb5 100644 --- a/libs/libloggers/loggers/OwnSplitChannel.cpp +++ b/libs/libloggers/loggers/OwnSplitChannel.cpp @@ -64,14 +64,16 @@ void OwnSplitChannel::log(const Poco::Message & msg) elem.thread_name = getThreadName(); elem.thread_number = msg_ext.thread_number; elem.os_thread_id = msg.getOsTid(); - elem.message = msg.getText(); + elem.query_id = msg_ext.query_id; + + elem.message = msg.getText(); + elem.logger_name = msg.getSource(); elem.level = msg.getPriority(); if (msg.getSourceFile() != nullptr) { elem.source_file = msg.getSourceFile(); } - elem.source_line = msg.getSourceLine(); text_log->add(elem); From 0f3a4a34b49849b12aedea840c9a76dc7531410a Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 22 Jul 2019 18:33:38 +0300 Subject: [PATCH 03/26] test + comment --- dbms/src/Interpreters/SystemLog.h | 2 +- dbms/src/Interpreters/TextLog.h | 2 -- .../0_stateless/00974_text_log_table_not_empty.reference | 1 + .../queries/0_stateless/00974_text_log_table_not_empty.sql | 2 ++ 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference create mode 100644 dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 83f4d6bbc12..03accf31cd9 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -74,7 +74,7 @@ struct SystemLogs std::shared_ptr query_log; /// Used to log queries. std::shared_ptr query_thread_log; /// Used to log query threads. std::shared_ptr part_log; /// Used to log operations with parts - std::shared_ptr text_log; + std::shared_ptr text_log; /// Used to save all text logs. String part_log_database; }; diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h index a8d76896b01..f56017b13bc 100644 --- a/dbms/src/Interpreters/TextLog.h +++ b/dbms/src/Interpreters/TextLog.h @@ -10,7 +10,6 @@ struct TextLogElement { time_t event_date{}; time_t event_time{}; - UInt32 microseconds; String thread_name; @@ -20,7 +19,6 @@ struct TextLogElement Message::Priority level = Message::PRIO_TRACE; String query_id; - String logger_name; String message; diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql new file mode 100644 index 00000000000..26bf82d62a5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -0,0 +1,2 @@ +SELECT count() > 0 +FROM system.text_log \ No newline at end of file From a80e066769c999f86b39a1e8f509709dcc275836 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 22 Jul 2019 19:04:15 +0300 Subject: [PATCH 04/26] better --- dbms/src/Interpreters/TextLog.cpp | 4 ---- dbms/src/Interpreters/TextLog.h | 1 - 2 files changed, 5 deletions(-) diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp index d11f6246319..5eb19958ab4 100644 --- a/dbms/src/Interpreters/TextLog.cpp +++ b/dbms/src/Interpreters/TextLog.cpp @@ -22,7 +22,6 @@ template <> struct NearestFieldTypeImpl { using Type = UInt64 Block TextLogElement::createBlock() { - auto priority_datatype = std::make_shared( DataTypeEnum8::Values { @@ -47,7 +46,6 @@ Block TextLogElement::createBlock() {std::make_shared(), "os_thread_id"}, {std::move(priority_datatype), "level"}, - {std::make_shared(), "query_id"}, {std::make_shared(), "logger_name"}, {std::make_shared(), "message"}, @@ -69,13 +67,11 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(event_time); columns[i++]->insert(microseconds); - // Thread info columns[i++]->insertData(thread_name.data(), thread_name.size()); columns[i++]->insert(thread_number); columns[i++]->insert(os_thread_id); columns[i++]->insert(level); - columns[i++]->insert(query_id); columns[i++]->insert(logger_name); columns[i++]->insert(message); diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h index f56017b13bc..31ce514c340 100644 --- a/dbms/src/Interpreters/TextLog.h +++ b/dbms/src/Interpreters/TextLog.h @@ -8,7 +8,6 @@ using Poco::Message; struct TextLogElement { - time_t event_date{}; time_t event_time{}; UInt32 microseconds; From daa7fc154fd4a4de01e5625efb2e1a1df8485431 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 22 Jul 2019 19:21:34 +0300 Subject: [PATCH 05/26] better includes --- dbms/src/Interpreters/TextLog.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp index 5eb19958ab4..884cc4568af 100644 --- a/dbms/src/Interpreters/TextLog.cpp +++ b/dbms/src/Interpreters/TextLog.cpp @@ -1,18 +1,10 @@ #include #include -#include -#include -#include #include #include #include #include -#include -#include -#include -#include #include -#include #include namespace DB From 61412e7f7492ef08b6efb963f9e64401f0073285 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 23 Jul 2019 11:22:38 +0300 Subject: [PATCH 06/26] changes after review + update config + resolve build fail --- dbms/src/Interpreters/TextLog.cpp | 16 ++++++++-------- dbms/tests/config/text_log.xml | 7 +++++++ docker/test/stateful/Dockerfile | 1 + docker/test/stateful_with_coverage/run.sh | 1 + docker/test/stateless/Dockerfile | 1 + docker/test/stateless_with_coverage/run.sh | 1 + libs/libloggers/loggers/OwnSplitChannel.cpp | 2 +- 7 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 dbms/tests/config/text_log.xml diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp index 884cc4568af..a6d21524c29 100644 --- a/dbms/src/Interpreters/TextLog.cpp +++ b/dbms/src/Interpreters/TextLog.cpp @@ -17,14 +17,14 @@ Block TextLogElement::createBlock() auto priority_datatype = std::make_shared( DataTypeEnum8::Values { - {"FATAL", static_cast(Message::PRIO_FATAL)}, - {"CRITICAL", static_cast(Message::PRIO_CRITICAL)}, - {"ERROR", static_cast(Message::PRIO_ERROR)}, - {"WARNING", static_cast(Message::PRIO_WARNING)}, - {"NOTICE", static_cast(Message::PRIO_NOTICE)}, - {"INFORMATION", static_cast(Message::PRIO_INFORMATION)}, - {"DEBUG", static_cast(Message::PRIO_DEBUG)}, - {"TRACE", static_cast(Message::PRIO_TRACE)} + {"Fatal", static_cast(Message::PRIO_FATAL)}, + {"Critical", static_cast(Message::PRIO_CRITICAL)}, + {"Error", static_cast(Message::PRIO_ERROR)}, + {"Warning", static_cast(Message::PRIO_WARNING)}, + {"Notice", static_cast(Message::PRIO_NOTICE)}, + {"Information", static_cast(Message::PRIO_INFORMATION)}, + {"Debug", static_cast(Message::PRIO_DEBUG)}, + {"Trace", static_cast(Message::PRIO_TRACE)} }); return diff --git a/dbms/tests/config/text_log.xml b/dbms/tests/config/text_log.xml new file mode 100644 index 00000000000..3699a23578c --- /dev/null +++ b/dbms/tests/config/text_log.xml @@ -0,0 +1,7 @@ + + + system + text_log
+ 7500 + + diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 466578c8b98..516d63fa330 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -19,6 +19,7 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh index 6ec0bfa155a..6253a07c745 100755 --- a/docker/test/stateful_with_coverage/run.sh +++ b/docker/test/stateful_with_coverage/run.sh @@ -45,6 +45,7 @@ chmod 777 -R /var/log/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index e2cd5eee933..eea48c3c032 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -39,6 +39,7 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh index 50082337757..5d63f9f49d0 100755 --- a/docker/test/stateless_with_coverage/run.sh +++ b/docker/test/stateless_with_coverage/run.sh @@ -47,6 +47,7 @@ chmod 777 -R /var/log/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ diff --git a/libs/libloggers/loggers/OwnSplitChannel.cpp b/libs/libloggers/loggers/OwnSplitChannel.cpp index c9f8f5e8bb5..926cf06d09a 100644 --- a/libs/libloggers/loggers/OwnSplitChannel.cpp +++ b/libs/libloggers/loggers/OwnSplitChannel.cpp @@ -63,7 +63,7 @@ void OwnSplitChannel::log(const Poco::Message & msg) elem.thread_name = getThreadName(); elem.thread_number = msg_ext.thread_number; - elem.os_thread_id = msg.getOsTid(); + elem.os_thread_id = CurrentThread::get().os_thread_id; elem.query_id = msg_ext.query_id; From b0ee2a4ce07a1bcd32a93ca405d8b6bd9b38736b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 23 Jul 2019 12:10:00 +0300 Subject: [PATCH 07/26] system flush logs added --- .../0_stateless/00974_text_log_table_not_empty.reference | 2 +- .../queries/0_stateless/00974_text_log_table_not_empty.sql | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference index 56a6051ca2b..d00491fd7e5 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -1 +1 @@ -1 \ No newline at end of file +1 diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql index 26bf82d62a5..9097dea4897 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -1,2 +1,4 @@ SELECT count() > 0 -FROM system.text_log \ No newline at end of file +FROM system.text_log + +SYSTEM FLUSH LOGS From 23dba8d025004bba4bf1740a029625a644f83fbf Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 23 Jul 2019 13:50:43 +0300 Subject: [PATCH 08/26] better test --- .../0_stateless/00974_text_log_table_not_empty.reference | 1 + .../0_stateless/00974_text_log_table_not_empty.sql | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference index d00491fd7e5..6ed281c757a 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -1 +1,2 @@ 1 +1 diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql index 9097dea4897..c6f1f536a53 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -1,4 +1,6 @@ -SELECT count() > 0 -FROM system.text_log +SELECT 1; -SYSTEM FLUSH LOGS +SYSTEM FLUSH LOGS; + +SELECT count() > 0 +FROM system.text_log; From 811e3ab241b60d3b78ec5e5f424dee95ed7eaa8e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 23 Jul 2019 17:50:38 +0300 Subject: [PATCH 09/26] style check --- dbms/src/Interpreters/Context.cpp | 6 +++--- dbms/src/Interpreters/Context.h | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index ab84fbc7364..6ae6bcb6d5b 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1702,14 +1702,14 @@ std::shared_ptr Context::getTextLog() return shared->system_logs->text_log; } - - + + std::shared_ptr Context::getTraceLog() { auto lock = getLock(); if (!shared->system_logs || !shared->system_logs->trace_log) - return nullptr; + return {}; return shared->system_logs->trace_log; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 3aa1422ffaa..2c5e0511efa 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -429,7 +429,6 @@ public: std::shared_ptr getQueryLog(); std::shared_ptr getQueryThreadLog(); std::shared_ptr getTextLog(); - std::shared_ptr getTraceLog(); /// Returns an object used to log opertaions with parts if it possible. From 38607af95fd82c08d8d08e8c441f8fd4ee19c2f5 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 24 Jul 2019 11:26:23 +0300 Subject: [PATCH 10/26] nothing interesting --- dbms/src/Interpreters/TextLog.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h index 31ce514c340..34f40ffe5f1 100644 --- a/dbms/src/Interpreters/TextLog.h +++ b/dbms/src/Interpreters/TextLog.h @@ -34,5 +34,4 @@ class TextLog : public SystemLog using SystemLog::SystemLog; }; - } From e382089c7fe4e4a3a587ea097f43b413fa192bff Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 24 Jul 2019 11:41:31 +0300 Subject: [PATCH 11/26] better test --- .../0_stateless/00974_text_log_table_not_empty.reference | 2 +- .../0_stateless/00974_text_log_table_not_empty.sql | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference index 6ed281c757a..0cc5d717d83 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -1,2 +1,2 @@ -1 +6103 1 diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql index c6f1f536a53..ee1469d968e 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -1,6 +1,9 @@ -SELECT 1; +SELECT 6103; SYSTEM FLUSH LOGS; -SELECT count() > 0 -FROM system.text_log; +SELECT count(query) > 0 +FROM system.text_log AS natural +INNER JOIN system.query_log USING (query_id) +WHERE query = 'SELECT 6103' + From df14bd6142c48be849b169f7994e7c3143b969b8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 24 Jul 2019 15:02:58 +0300 Subject: [PATCH 12/26] test with sleep --- .../0_stateless/00974_text_log_table_not_empty.reference | 1 + .../queries/0_stateless/00974_text_log_table_not_empty.sql | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference index 0cc5d717d83..0c56a79a1c5 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -1,2 +1,3 @@ 6103 +0 1 diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql index ee1469d968e..8cf62c9cdbb 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -1,5 +1,7 @@ SELECT 6103; +SELECT sleep(1); + SYSTEM FLUSH LOGS; SELECT count(query) > 0 From f637e22eb6e3712729ddf40dc1af53befff5fb17 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 25 Jul 2019 11:50:16 +0300 Subject: [PATCH 13/26] tests... --- .../00974_text_log_table_not_empty.reference | 1 - .../0_stateless/00974_text_log_table_not_empty.sql | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference index 0c56a79a1c5..0cc5d717d83 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.reference @@ -1,3 +1,2 @@ 6103 -0 1 diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql index 8cf62c9cdbb..7f87ac29908 100644 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql @@ -1,11 +1,9 @@ SELECT 6103; -SELECT sleep(1); - SYSTEM FLUSH LOGS; -SELECT count(query) > 0 -FROM system.text_log AS natural -INNER JOIN system.query_log USING (query_id) -WHERE query = 'SELECT 6103' +SELECT count() > 0 +FROM system.text_log +WHERE position(system.text_log.message, 'SELECT 6103') > 0 + From 795eebb5ad8d70e2077361582d7cc655e103807c Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 25 Jul 2019 12:39:36 +0300 Subject: [PATCH 14/26] stable tests --- .../0_stateless/00974_text_log_table_not_empty.sh | 8 ++++++++ .../0_stateless/00974_text_log_table_not_empty.sql | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) create mode 100755 dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh delete mode 100644 dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh new file mode 100755 index 00000000000..7eaf2c35674 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="SELECT 6103" +${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" +${CLICKHOUSE_CLIENT} --query="SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0" diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql deleted file mode 100644 index 7f87ac29908..00000000000 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT 6103; - -SYSTEM FLUSH LOGS; - -SELECT count() > 0 -FROM system.text_log -WHERE position(system.text_log.message, 'SELECT 6103') > 0 - - From e5df077c21a339338cbf962b3d8e5eb1f67415d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 26 Jul 2019 02:00:18 +0300 Subject: [PATCH 15/26] Added example in config --- dbms/programs/server/config.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index a8f37a2cd78..c09913cbd87 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -322,6 +322,14 @@ --> +