From 22870c58695a55b989d34c7a56208355c56ba545 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 11:47:18 +0300 Subject: [PATCH 1/4] few files --- dbms/src/Interpreters/MetricLog.cpp | 88 +++++++++++++++++++++++++++++ dbms/src/Interpreters/MetricLog.h | 24 ++++++++ dbms/src/Interpreters/SystemLog.cpp | 3 + dbms/src/Interpreters/SystemLog.h | 2 + 4 files changed, 117 insertions(+) create mode 100644 dbms/src/Interpreters/MetricLog.cpp create mode 100644 dbms/src/Interpreters/MetricLog.h diff --git a/dbms/src/Interpreters/MetricLog.cpp b/dbms/src/Interpreters/MetricLog.cpp new file mode 100644 index 00000000000..e968b568c06 --- /dev/null +++ b/dbms/src/Interpreters/MetricLog.cpp @@ -0,0 +1,88 @@ +#include +#include +#include + +namespace DB +{ + +Block MetricLogElement::createBlock() +{ + ColumnsWithTypeAndName columns_with_type_and_name; + //ProfileEvents + for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) + { + UInt64 value = ProfileEvents::global_counters[i]; + + if (0 != value) + { + std::string name; + name += "ProfileEvent_"; + name += ProfileEvents::getName(ProfileEvents::Event(i)); + columns_with_type_and_name.emplace_back(std::make_shared(), name); + } + } + + //CurrentMetrics + for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) + { + UInt64 value = CurrentMetrics::values[i]; + + if (0 != value) + { + std::string name; + name += "CurrentMetric_"; + name += CurrentMetrics::getName(ProfileEvents::Event(i)); + columns_with_type_and_name.emplace_back(std::make_shared(), name); + } + } + + //AsyncMetrics + auto async_metrics_values = async_metrics_ptr.getValues(); + for (const auto & name_value : async_metrics_values) + { + std::string name; + name += "AsynchronousMetrics_"; + name += name_value.first; + columns_with_type_and_name.emplace_back(std::make_shared(), name); + } + + return Block(columns_with_type_and_name); +} + +void MetricLogElement::appendToBlock(Block & block) const +{ + MutableColumns columns = block.mutateColumns(); + + size_t iter = 0; + + //ProfileEvents + for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) + { + UInt64 value = ProfileEvents::global_counters[i]; + + if (0 != value) + { + columns[iter++]->insert(value); + } + } + + //CurrentMetrics + for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) + { + UInt64 value = CurrentMetrics::values[i]; + + if (0 != value) + { + columns[iter++]->insert(value); + } + } + + //AsyncMetrics + auto async_metrics_values = async_metrics_ptr.getValues(); + for (const auto & name_value : async_metrics_values) + { + columns[iter++]->insert(name_value.second); + } +} + +} diff --git a/dbms/src/Interpreters/MetricLog.h b/dbms/src/Interpreters/MetricLog.h new file mode 100644 index 00000000000..ed97578afd5 --- /dev/null +++ b/dbms/src/Interpreters/MetricLog.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include + +namespace DB +{ + +using Poco::Message; + +struct MetricLogElement +{ + std::shared_ptr async_metrics_ptr{nullptr}; + + static std::string name() { return "MetricLog"; } + static Block createBlock(); + void appendToBlock(Block & block) const; +}; + +class MetricLog : public SystemLog +{ + using SystemLog::SystemLog; +}; + +} diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index f1f65dfe883..a070c43c883 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -48,6 +48,7 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi part_log = createSystemLog(global_context, "system", "part_log", config, "part_log"); trace_log = createSystemLog(global_context, "system", "trace_log", config, "trace_log"); text_log = createSystemLog(global_context, "system", "text_log", config, "text_log"); + metric_log = createSystemLog(global_context, "system", "metric_log", config, "metric_log"); part_log_database = config.getString("part_log.database", "system"); } @@ -70,6 +71,8 @@ void SystemLogs::shutdown() trace_log->shutdown(); if (text_log) text_log->shutdown(); + if (metric_log) + metric_log->shutdown(); } } diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 3dd329d577b..b00f77b7622 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -61,6 +61,7 @@ class QueryThreadLog; class PartLog; class TextLog; class TraceLog; +class MetricLog; /// System logs should be destroyed in destructor of the last Context and before tables, /// because SystemLog destruction makes insert query while flushing data into underlying tables @@ -76,6 +77,7 @@ struct SystemLogs std::shared_ptr part_log; /// Used to log operations with parts std::shared_ptr trace_log; /// Used to log traces from query profiler std::shared_ptr text_log; /// Used to log all text messages. + std::shared_ptr metric_log; /// Used to log all metrics. String part_log_database; }; From 3f63d41baa8cbf2f70d5d8fdc7f6adf5be4462d1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 16:35:20 +0300 Subject: [PATCH 2/4] submodules --- contrib/simdjson | 2 +- contrib/snappy | 2 +- contrib/thrift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/simdjson b/contrib/simdjson index e3f6322af76..3fb82502f7f 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit e3f6322af762213ff2087ce3366bf9541c7fd355 +Subproject commit 3fb82502f7f19a098006e7ff74c9b6e7c7dd4a84 diff --git a/contrib/snappy b/contrib/snappy index 3f194acb57e..156cd8939c5 160000 --- a/contrib/snappy +++ b/contrib/snappy @@ -1 +1 @@ -Subproject commit 3f194acb57e0487531c96b97af61dcbd025a78a3 +Subproject commit 156cd8939c5fba7fa68ae08db843377ecc07b4b5 diff --git a/contrib/thrift b/contrib/thrift index 010ccf0a0c7..74d6d9d3d64 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982 +Subproject commit 74d6d9d3d6400d1672f48b4acf5bc7d1260ad96d From 3d2e9867cc8c398f21a1222726617d79e1d66d59 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 16:38:30 +0300 Subject: [PATCH 3/4] arrow submodule --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index 87ac6fddaf2..93688e8c1fa 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 87ac6fddaf21d0b4ee8b8090533ff293db0da1b4 +Subproject commit 93688e8c1fa2f22d46394c548a9edbd3d2d7c62d From 46a5ec5c16ed274956cc2051d0d2da213d975051 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 17:56:58 +0300 Subject: [PATCH 4/4] some changes --- dbms/src/Interpreters/MetricLog.cpp | 17 ----------------- dbms/src/Interpreters/MetricLog.h | 2 -- dbms/src/Interpreters/SystemLog.cpp | 6 ++++++ dbms/src/Interpreters/SystemLog.h | 2 ++ 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/dbms/src/Interpreters/MetricLog.cpp b/dbms/src/Interpreters/MetricLog.cpp index e968b568c06..8f421a65dea 100644 --- a/dbms/src/Interpreters/MetricLog.cpp +++ b/dbms/src/Interpreters/MetricLog.cpp @@ -36,16 +36,6 @@ Block MetricLogElement::createBlock() } } - //AsyncMetrics - auto async_metrics_values = async_metrics_ptr.getValues(); - for (const auto & name_value : async_metrics_values) - { - std::string name; - name += "AsynchronousMetrics_"; - name += name_value.first; - columns_with_type_and_name.emplace_back(std::make_shared(), name); - } - return Block(columns_with_type_and_name); } @@ -76,13 +66,6 @@ void MetricLogElement::appendToBlock(Block & block) const columns[iter++]->insert(value); } } - - //AsyncMetrics - auto async_metrics_values = async_metrics_ptr.getValues(); - for (const auto & name_value : async_metrics_values) - { - columns[iter++]->insert(name_value.second); - } } } diff --git a/dbms/src/Interpreters/MetricLog.h b/dbms/src/Interpreters/MetricLog.h index ed97578afd5..5e034463d0e 100644 --- a/dbms/src/Interpreters/MetricLog.h +++ b/dbms/src/Interpreters/MetricLog.h @@ -9,8 +9,6 @@ using Poco::Message; struct MetricLogElement { - std::shared_ptr async_metrics_ptr{nullptr}; - static std::string name() { return "MetricLog"; } static Block createBlock(); void appendToBlock(Block & block) const; diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index a070c43c883..b8e8c5d7c1c 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -75,4 +76,9 @@ void SystemLogs::shutdown() metric_log->shutdown(); } +void SystemLogs::threadFunction() +{ + +} + } diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index b00f77b7622..f42febdf090 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -79,6 +79,8 @@ struct SystemLogs std::shared_ptr text_log; /// Used to log all text messages. std::shared_ptr metric_log; /// Used to log all metrics. + void threadFunction(); + String part_log_database; };