From 670e98fa920f8af28d85a69a2a009f4063363c7c Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 21 Mar 2017 22:08:09 +0300 Subject: [PATCH] allow several targets (#603) * allow several targets * fix * fix * Adjustable parts * changelog version * fix * changelog * Style fixes * attachSystemTables * config describe * fixes * fixes --- CHANGELOG.md | 6 ++ .../DB/Common/getMultipleKeysFromConfig.h | 16 +++++ .../DB/Storages/System/attachSystemTables.h | 13 ++++ .../DB/Storages/System/attach_system_tables.h | 13 ---- dbms/src/Common/getMultipleKeysFromConfig.cpp | 21 ++++++ dbms/src/Server/LocalServer.cpp | 4 +- dbms/src/Server/MetricsTransmitter.cpp | 67 ++++++++++++------- dbms/src/Server/MetricsTransmitter.h | 18 ++--- dbms/src/Server/Server.cpp | 45 +++++++------ dbms/src/Server/config.d/no_graphite.xml | 3 - dbms/src/Server/config.xml | 31 +++++++-- ...stem_tables.cpp => attachSystemTables.cpp} | 10 +-- doc/developers/style_ru.md | 2 + format_sources | 2 +- .../include/common/iostream_debug_helpers.h | 24 +++++++ libs/libdaemon/include/daemon/BaseDaemon.h | 21 ++++-- libs/libdaemon/src/BaseDaemon.cpp | 14 ++-- 17 files changed, 213 insertions(+), 97 deletions(-) create mode 100644 dbms/include/DB/Common/getMultipleKeysFromConfig.h create mode 100644 dbms/include/DB/Storages/System/attachSystemTables.h delete mode 100644 dbms/include/DB/Storages/System/attach_system_tables.h create mode 100644 dbms/src/Common/getMultipleKeysFromConfig.cpp delete mode 100644 dbms/src/Server/config.d/no_graphite.xml rename dbms/src/Storages/System/{attach_system_tables.cpp => attachSystemTables.cpp} (89%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c25f6db99..167328dfbf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ +## [1.1.54189](https://github.com/yandex/Clickhouse/tree/v1.1.54189-testing) (2017-03-17) +[Full Changelog](https://github.com/yandex/Clickhouse/compare/v1.1.54188-stable...v1.1.54189-testing) + +- Config: Allow define several graphite blocks, graphite.interval=60 option added. use_graphite option deleted. + + ## [1.1.54181](https://github.com/yandex/Clickhouse/tree/v1.1.54181-testing) (2017-03-10) [Full Changelog](https://github.com/yandex/Clickhouse/compare/v1.1.54165-stable...v1.1.54181-testing) diff --git a/dbms/include/DB/Common/getMultipleKeysFromConfig.h b/dbms/include/DB/Common/getMultipleKeysFromConfig.h new file mode 100644 index 00000000000..f6c7732c61a --- /dev/null +++ b/dbms/include/DB/Common/getMultipleKeysFromConfig.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace Poco +{ +namespace Util +{ + class AbstractConfiguration; +} +} +namespace DB +{ +/// get all internal key names for given key +std::vector getMultipleKeysFromConfig(Poco::Util::AbstractConfiguration & config, const std::string & root, const std::string & name); +} diff --git a/dbms/include/DB/Storages/System/attachSystemTables.h b/dbms/include/DB/Storages/System/attachSystemTables.h new file mode 100644 index 00000000000..7928f763a01 --- /dev/null +++ b/dbms/include/DB/Storages/System/attachSystemTables.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace DB +{ +class Context; +class AsynchronousMetrics; + +void attachSystemTablesServer(DatabasePtr system_database, Context * global_context, bool has_zookeeper); +void attachSystemTablesLocal(DatabasePtr system_database); +void attachSystemTablesAsync(DatabasePtr system_database, AsynchronousMetrics & async_metrics); +} diff --git a/dbms/include/DB/Storages/System/attach_system_tables.h b/dbms/include/DB/Storages/System/attach_system_tables.h deleted file mode 100644 index 78b6c54e720..00000000000 --- a/dbms/include/DB/Storages/System/attach_system_tables.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include - -namespace DB -{ -class Context; -class AsynchronousMetrics; - -void attach_system_tables_server(DatabasePtr system_database, Context * global_context, bool has_zookeeper); -void attach_system_tables_local(DatabasePtr system_database); -void attach_system_tables_async(DatabasePtr system_database, AsynchronousMetrics & async_metrics); -} diff --git a/dbms/src/Common/getMultipleKeysFromConfig.cpp b/dbms/src/Common/getMultipleKeysFromConfig.cpp new file mode 100644 index 00000000000..e67203d7eef --- /dev/null +++ b/dbms/src/Common/getMultipleKeysFromConfig.cpp @@ -0,0 +1,21 @@ +#include + +#include +#include + +namespace DB +{ +std::vector getMultipleKeysFromConfig(Poco::Util::AbstractConfiguration & config, const std::string & root, const std::string & name) +{ + std::vector values; + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(root, config_keys); + for (const auto & key : config_keys) + { + if (key != name && !(startsWith(key.data(), name + "[") && endsWith(key.data(), "]"))) + continue; + values.emplace_back(key); + } + return values; +} +} diff --git a/dbms/src/Server/LocalServer.cpp b/dbms/src/Server/LocalServer.cpp index 4875cd15f13..6e097a6a146 100644 --- a/dbms/src/Server/LocalServer.cpp +++ b/dbms/src/Server/LocalServer.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -379,7 +379,7 @@ void LocalServer::attachSystemTables() context->addDatabase("system", system_database); } - attach_system_tables_local(system_database); + attachSystemTablesLocal(system_database); } diff --git a/dbms/src/Server/MetricsTransmitter.cpp b/dbms/src/Server/MetricsTransmitter.cpp index eee9b16e4fd..010563b0f62 100644 --- a/dbms/src/Server/MetricsTransmitter.cpp +++ b/dbms/src/Server/MetricsTransmitter.cpp @@ -1,15 +1,15 @@ #include "MetricsTransmitter.h" +#include +#include #include -#include #include #include +#include #include - namespace DB { - MetricsTransmitter::~MetricsTransmitter() { try @@ -32,13 +32,19 @@ MetricsTransmitter::~MetricsTransmitter() void MetricsTransmitter::run() { - setThreadName("MetricsTransmit"); + auto & config = Poco::Util::Application::instance().config(); + auto interval = config.getInt(config_name + ".interval", 60); - /// Next minute at 00 seconds. To avoid time drift and transmit values exactly each minute. - const auto get_next_minute = [] - { - return std::chrono::time_point_cast( - std::chrono::system_clock::now() + std::chrono::minutes(1)); + const std::string thread_name = "MericsTrns " + std::to_string(interval) + "s"; + setThreadName(thread_name.c_str()); + + const auto get_next_time = [](size_t seconds) { + /// To avoid time drift and transmit values exactly each interval: + /// next time aligned to system seconds + /// (60s -> every minute at 00 seconds, 5s -> every minute:[00, 05, 15 ... 55]s, 3600 -> every hour:00:00 + return std::chrono::system_clock::time_point( + (std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) / seconds) * seconds + + std::chrono::seconds(seconds)); }; std::vector prev_counters(ProfileEvents::end()); @@ -47,7 +53,7 @@ void MetricsTransmitter::run() while (true) { - if (cond.wait_until(lock, get_next_minute(), [this] { return quit; })) + if (cond.wait_until(lock, get_next_time(interval), [this] { return quit; })) break; transmit(prev_counters); @@ -57,35 +63,46 @@ void MetricsTransmitter::run() void MetricsTransmitter::transmit(std::vector & prev_counters) { + auto & config = Poco::Util::Application::instance().config(); auto async_metrics_values = async_metrics.getValues(); GraphiteWriter::KeyValueVector key_vals{}; key_vals.reserve(ProfileEvents::end() + CurrentMetrics::end() + async_metrics_values.size()); - for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) - { - const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed); - const auto counter_increment = counter - prev_counters[i]; - prev_counters[i] = counter; - std::string key {ProfileEvents::getDescription(static_cast(i))}; - key_vals.emplace_back(profile_events_path_prefix + key, counter_increment); + if (config.getBool(config_name + ".events", true)) + { + for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) + { + const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed); + const auto counter_increment = counter - prev_counters[i]; + prev_counters[i] = counter; + + std::string key{ProfileEvents::getDescription(static_cast(i))}; + key_vals.emplace_back(profile_events_path_prefix + key, counter_increment); + } } - for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) + if (config.getBool(config_name + ".metrics", true)) { - const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed); + for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) + { + const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed); - std::string key {CurrentMetrics::getDescription(static_cast(i))}; - key_vals.emplace_back(current_metrics_path_prefix + key, value); + std::string key{CurrentMetrics::getDescription(static_cast(i))}; + key_vals.emplace_back(current_metrics_path_prefix + key, value); + } } - for (const auto & name_value : async_metrics_values) + if (config.getBool(config_name + ".asynchronous_metrics", true)) { - key_vals.emplace_back(asynchronous_metrics_path_prefix + name_value.first, name_value.second); + for (const auto & name_value : async_metrics_values) + { + key_vals.emplace_back(asynchronous_metrics_path_prefix + name_value.first, name_value.second); + } } - BaseDaemon::instance().writeToGraphite(key_vals); + if (key_vals.size()) + BaseDaemon::instance().writeToGraphite(key_vals, config_name); } - } diff --git a/dbms/src/Server/MetricsTransmitter.h b/dbms/src/Server/MetricsTransmitter.h index ffa1cfc599b..b3cf5757bf5 100644 --- a/dbms/src/Server/MetricsTransmitter.h +++ b/dbms/src/Server/MetricsTransmitter.h @@ -1,16 +1,15 @@ #pragma once -#include -#include -#include #include - +#include +#include +#include +#include #include namespace DB { - class AsynchronousMetrics; /** Automatically sends @@ -22,7 +21,10 @@ class AsynchronousMetrics; class MetricsTransmitter { public: - MetricsTransmitter(const AsynchronousMetrics & async_metrics_) : async_metrics(async_metrics_) {} + MetricsTransmitter(const AsynchronousMetrics & async_metrics, const std::string & config_name) + : async_metrics{async_metrics}, config_name{config_name} + { + } ~MetricsTransmitter(); private: @@ -30,15 +32,15 @@ private: void transmit(std::vector & prev_counters); const AsynchronousMetrics & async_metrics; + const std::string config_name; bool quit = false; std::mutex mutex; std::condition_variable cond; - std::thread thread {&MetricsTransmitter::run, this}; + std::thread thread{&MetricsTransmitter::run, this}; static constexpr auto profile_events_path_prefix = "ClickHouse.ProfileEvents."; static constexpr auto current_metrics_path_prefix = "ClickHouse.Metrics."; static constexpr auto asynchronous_metrics_path_prefix = "ClickHouse.AsynchronousMetrics."; }; - } diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 0540f3b991c..566c45f1f78 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -13,9 +13,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -23,7 +25,7 @@ #include #include #include -#include +#include #include "ConfigReloader.h" #include "HTTPHandler.h" #include "InterserverIOHTTPHandler.h" @@ -32,6 +34,7 @@ #include "StatusFile.h" #include "TCPHandler.h" + namespace DB { namespace ErrorCodes @@ -221,7 +224,7 @@ int Server::main(const std::vector & args) { auto old_configuration = loaded_config.configuration; loaded_config = ConfigProcessor().loadConfigWithZooKeeperIncludes( - config_path, main_config_zk_node_cache, /* fallback_to_preprocessed = */ true); + config_path, main_config_zk_node_cache, /* fallback_to_preprocessed = */ true); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -321,11 +324,11 @@ int Server::main(const std::vector & args) /// Initialize main config reloader. std::string include_from_path = config().getString("include_from", "/etc/metrika.xml"); - auto main_config_reloader = std::make_unique( - config_path, include_from_path, - std::move(main_config_zk_node_cache), - [&](ConfigurationPtr config) { global_context->setClustersConfig(config); }, - /* already_loaded = */ true); + auto main_config_reloader = std::make_unique(config_path, + include_from_path, + std::move(main_config_zk_node_cache), + [&](ConfigurationPtr config) { global_context->setClustersConfig(config); }, + /* already_loaded = */ true); /// Initialize users config reloader. std::string users_config_path = config().getString("users_config", config_path); @@ -337,11 +340,11 @@ int Server::main(const std::vector & args) if (Poco::File(config_dir + users_config_path).exists()) users_config_path = config_dir + users_config_path; } - auto users_config_reloader = std::make_unique( - users_config_path, include_from_path, - zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), - [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, - /* already_loaded = */ false); + auto users_config_reloader = std::make_unique(users_config_path, + include_from_path, + zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), + [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, + /* already_loaded = */ false); /// Limit on total number of coucurrently executed queries. global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0)); @@ -385,7 +388,7 @@ int Server::main(const std::vector & args) DatabasePtr system_database = global_context->getDatabase("system"); - attach_system_tables_server(system_database, global_context.get(), has_zookeeper); + attachSystemTablesServer(system_database, global_context.get(), has_zookeeper); bool has_resharding_worker = false; if (has_zookeeper && config().has("resharding")) @@ -425,12 +428,8 @@ int Server::main(const std::vector & args) std::vector> servers; std::vector listen_hosts; - Poco::Util::AbstractConfiguration::Keys config_keys; - config().keys("", config_keys); - for (const auto & key : config_keys) + for (const auto & key : DB::getMultipleKeysFromConfig(config(), "", "listen_host")) { - if (!startsWith(key.data(), "listen_host")) - continue; listen_hosts.emplace_back(config().getString(key)); } @@ -585,10 +584,14 @@ int Server::main(const std::vector & args) /// This object will periodically calculate some metrics. AsynchronousMetrics async_metrics(*global_context); - attach_system_tables_async(system_database, async_metrics); + attachSystemTablesAsync(system_database, async_metrics); + + std::vector> metrics_transmitters; + for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) + { + metrics_transmitters.emplace_back(std::make_unique(async_metrics, graphite_key)); + } - const auto metrics_transmitter - = config().getBool("use_graphite", true) ? std::make_unique(async_metrics) : nullptr; waitForTerminationRequest(); } diff --git a/dbms/src/Server/config.d/no_graphite.xml b/dbms/src/Server/config.d/no_graphite.xml deleted file mode 100644 index 89f76aa3d4f..00000000000 --- a/dbms/src/Server/config.d/no_graphite.xml +++ /dev/null @@ -1,3 +0,0 @@ - - false - diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml index 5cdb4995b81..c78f632cce4 100644 --- a/dbms/src/Server/config.xml +++ b/dbms/src/Server/config.xml @@ -138,15 +138,36 @@ 3600 - - false - - + + diff --git a/dbms/src/Storages/System/attach_system_tables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp similarity index 89% rename from dbms/src/Storages/System/attach_system_tables.cpp rename to dbms/src/Storages/System/attachSystemTables.cpp index f798ba393a4..ce83787111a 100644 --- a/dbms/src/Storages/System/attach_system_tables.cpp +++ b/dbms/src/Storages/System/attachSystemTables.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -23,7 +23,7 @@ namespace DB { -void attach_system_tables_local(DatabasePtr system_database) +void attachSystemTablesLocal(DatabasePtr system_database) { system_database->attachTable("one", StorageSystemOne::create("one")); system_database->attachTable("numbers", StorageSystemNumbers::create("numbers")); @@ -37,9 +37,9 @@ void attach_system_tables_local(DatabasePtr system_database) system_database->attachTable("build_options", StorageSystemBuildOptions::create("build_options")); } -void attach_system_tables_server(DatabasePtr system_database, Context * global_context, bool has_zookeeper) +void attachSystemTablesServer(DatabasePtr system_database, Context * global_context, bool has_zookeeper) { - attach_system_tables_local(system_database); + attachSystemTablesLocal(system_database); system_database->attachTable("parts", StorageSystemParts::create("parts")); system_database->attachTable("processes", StorageSystemProcesses::create("processes")); system_database->attachTable("metrics", StorageSystemMetrics::create("metrics")); @@ -54,7 +54,7 @@ void attach_system_tables_server(DatabasePtr system_database, Context * global_c system_database->attachTable("zookeeper", StorageSystemZooKeeper::create("zookeeper")); } -void attach_system_tables_async(DatabasePtr system_database, AsynchronousMetrics & async_metrics) +void attachSystemTablesAsync(DatabasePtr system_database, AsynchronousMetrics & async_metrics) { system_database->attachTable("asynchronous_metrics", StorageSystemAsynchronousMetrics::create("asynchronous_metrics", async_metrics)); } diff --git a/doc/developers/style_ru.md b/doc/developers/style_ru.md index dcc627239e5..8aa7cad62df 100644 --- a/doc/developers/style_ru.md +++ b/doc/developers/style_ru.md @@ -15,6 +15,8 @@ ## 1. Форматирование +0. Большую часть форматирования сделает автоматически clang-format. Инструкция для подключения clang-format в kdevelop описана в файле format_sources + 1. Отступы табами. Настройте среду разработки так, чтобы таб был по ширине равен четырём символам. 2. Открывающая фигурная скобка на новой, отдельной строке. (Закрывающая - тоже.) diff --git a/format_sources b/format_sources index 62b523c40ff..e77009ec4db 100644 --- a/format_sources +++ b/format_sources @@ -1,2 +1,2 @@ -# Settings -> Configure KDevelop -> Source Formatter -> C++ ; Custom Script Formatter ; Kdevelop: kdev_format_source +# Settings -> Configure KDevelop -> Source Formatter -> [C++, C] ; Custom Script Formatter ; Kdevelop: kdev_format_source *.c *.cpp *.h *.hpp: mv $TMPFILE $TMPFILE.tmp; cat $TMPFILE.tmp | clang-format -style=file -assume-filename=`pwd`/.clang-format > $TMPFILE diff --git a/libs/libcommon/include/common/iostream_debug_helpers.h b/libs/libcommon/include/common/iostream_debug_helpers.h index 0e01ee896ec..4f311353845 100644 --- a/libs/libcommon/include/common/iostream_debug_helpers.h +++ b/libs/libcommon/include/common/iostream_debug_helpers.h @@ -85,5 +85,29 @@ std::ostream & operator<<(std::ostream & stream, const std::pair & what) return stream; } +#include + +template +std::ostream & operator<<(std::ostream & stream, const std::ratio & what) +{ + stream << "ratio"; + return stream; +} + +#include +template +std::ostream & operator<<(std::ostream & stream, const std::chrono::duration & what) +{ + stream << "chrono::duration{" << what.count() << "}"; + return stream; +} + +template +std::ostream & operator<<(std::ostream & stream, const std::chrono::time_point & what) +{ + stream << "chrono::time_point{" << what.time_since_epoch() << "}"; + return stream; +} + // TODO: add more types diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h index 494e7eac9a8..24d8ed5f79b 100644 --- a/libs/libdaemon/include/daemon/BaseDaemon.h +++ b/libs/libdaemon/include/daemon/BaseDaemon.h @@ -107,18 +107,27 @@ public: /// root_path по умолчанию one_min /// key - лучше группировать по смыслу. Например "meminfo.cached" или "meminfo.free", "meminfo.total" template - void writeToGraphite(const std::string & key, const T & value, time_t timestamp = 0, const std::string & custom_root_path = "") + void writeToGraphite(const std::string & key, const T & value, const std::string & config_name = "graphite", time_t timestamp = 0, const std::string & custom_root_path = "") { - graphite_writer->write(key, value, timestamp, custom_root_path); + auto writer = getGraphiteWriter(config_name); + if (writer) + writer->write(key, value, timestamp, custom_root_path); } template - void writeToGraphite(const GraphiteWriter::KeyValueVector & key_vals, time_t timestamp = 0, const std::string & custom_root_path = "") + void writeToGraphite(const GraphiteWriter::KeyValueVector & key_vals, const std::string & config_name = "graphite", time_t timestamp = 0, const std::string & custom_root_path = "") { - graphite_writer->write(key_vals, timestamp, custom_root_path); + auto writer = getGraphiteWriter(config_name); + if (writer) + writer->write(key_vals, timestamp, custom_root_path); } - GraphiteWriter * getGraphiteWriter() { return graphite_writer.get(); } + GraphiteWriter * getGraphiteWriter(const std::string & config_name = "graphite") + { + if (graphite_writers.count(config_name)) + return graphite_writers[config_name].get(); + return nullptr; + } std::experimental::optional getLayer() const { @@ -196,7 +205,7 @@ protected: Poco::AutoPtr error_log_file; Poco::AutoPtr syslog_channel; - std::unique_ptr graphite_writer; + std::map> graphite_writers; std::experimental::optional layer; diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 18f33b901b2..1a06dc799d6 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -7,7 +7,6 @@ #include #include #include - #include #include #include @@ -17,16 +16,13 @@ #define _XOPEN_SOURCE #endif #include - #include - #include #include - #include #include - #include +#include #include #include #include @@ -48,13 +44,12 @@ #include #include #include - #include #include #include #include #include - +#include #include #include @@ -792,7 +787,10 @@ void BaseDaemon::initialize(Application& self) signal_listener.reset(new SignalListener(*this)); signal_listener_thread.start(*signal_listener); - graphite_writer.reset(new GraphiteWriter("graphite")); + for (const auto & key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) + { + graphite_writers.emplace(key, std::make_unique(key)); + } } void BaseDaemon::logRevision() const