From 8aebb9396fd4a4c654a634b30ac4b1c6ad5341e6 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Thu, 28 Jul 2016 11:06:04 +0300 Subject: [PATCH 1/4] deduplicatord: supported layers [#MOBMET-3538] --- libs/libdaemon/include/daemon/GraphiteWriter.h | 5 ++++- libs/libdaemon/src/GraphiteWriter.cpp | 12 ++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/libs/libdaemon/include/daemon/GraphiteWriter.h b/libs/libdaemon/include/daemon/GraphiteWriter.h index d5f180ba12e..6bbb17ece40 100644 --- a/libs/libdaemon/include/daemon/GraphiteWriter.h +++ b/libs/libdaemon/include/daemon/GraphiteWriter.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -34,7 +35,9 @@ public: /// Для облачных демонов удобней использовать /// путь вида prefix.environment.layer.daemon_name.metrica - static std::string getPerLayerPath(const std::string & prefix = "one_min"); + static std::string getPerLayerPath( + const std::string & prefix = "one_min", + const boost::optional layer = {}); /// возвращает путь root_path.server_name static std::string getPerServerPath(const std::string & server_name, const std::string & root_path = "one_min"); diff --git a/libs/libdaemon/src/GraphiteWriter.cpp b/libs/libdaemon/src/GraphiteWriter.cpp index 90946142713..662544db11a 100644 --- a/libs/libdaemon/src/GraphiteWriter.cpp +++ b/libs/libdaemon/src/GraphiteWriter.cpp @@ -49,7 +49,7 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin root_path += "." + sub_path; } -std::string getPostfix() +std::string getPostfix(const boost::optional layer) { /// Угадываем имя среды по имени машинки /// машинки имеют имена вида example01dt.yandex.ru @@ -73,7 +73,9 @@ std::string getPostfix() const BaseDaemon & daemon = BaseDaemon::instance(); - if (daemon.getLayer()) + if (layer) + path_full << "layer" << std::setfill('0') << std::setw(3) << *layer << "."; + else if (daemon.getLayer()) path_full << "layer" << std::setfill('0') << std::setw(3) << *daemon.getLayer() << "."; /// Когда несколько демонов запускается на одной машине @@ -88,9 +90,11 @@ std::string getPostfix() return path_full.str(); } -std::string GraphiteWriter::getPerLayerPath(const std::string & prefix) +std::string GraphiteWriter::getPerLayerPath( + const std::string & prefix, + const boost::optional layer) { - const std::string static postfix = getPostfix(); + const std::string static postfix = getPostfix(layer); return prefix + "." + postfix; } From ab78f8d0bf1988009cca6729e7eeb30674eba324 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Fri, 29 Jul 2016 10:58:26 +0300 Subject: [PATCH 2/4] GraphiteWriter: fixed invalid storage type of variable [#MOBMET-3538] --- libs/libdaemon/src/GraphiteWriter.cpp | 38 +++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/libs/libdaemon/src/GraphiteWriter.cpp b/libs/libdaemon/src/GraphiteWriter.cpp index 662544db11a..d65259ad98a 100644 --- a/libs/libdaemon/src/GraphiteWriter.cpp +++ b/libs/libdaemon/src/GraphiteWriter.cpp @@ -49,27 +49,38 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin root_path += "." + sub_path; } -std::string getPostfix(const boost::optional layer) +/// Угадываем имя среды по имени машинки +/// машинки имеют имена вида example01dt.yandex.ru +/// t - test +/// dev - development +/// никакого суффикса - production +std::string getEnvironment() { - /// Угадываем имя среды по имени машинки - /// машинки имеют имена вида example01dt.yandex.ru - /// t - test - /// dev - development - /// никакого суффикса - production - - std::stringstream path_full; - std::string hostname = Poco::Net::DNS::hostName(); hostname = hostname.substr(0, hostname.find('.')); const std::string development_suffix = "dev"; if (hostname.back() == 't') - path_full << "test."; + { + return "test"; + } else if (hostname.size() > development_suffix.size() && hostname.substr(hostname.size() - development_suffix.size()) == development_suffix) - path_full << "development."; + { + return "development"; + } else - path_full << "production."; + { + return "production"; + } +} + +std::string getPostfix(const boost::optional layer) +{ + static const std::string environment = getEnvironment(); + + std::stringstream path_full; + path_full << environment << "."; const BaseDaemon & daemon = BaseDaemon::instance(); @@ -94,8 +105,7 @@ std::string GraphiteWriter::getPerLayerPath( const std::string & prefix, const boost::optional layer) { - const std::string static postfix = getPostfix(layer); - return prefix + "." + postfix; + return prefix + "." + getPostfix(layer); } std::string GraphiteWriter::getPerServerPath(const std::string & server_name, const std::string & root_path) From 1bef561c328a9c6367d1459f45a865c022e86c52 Mon Sep 17 00:00:00 2001 From: Alexey Vasiliev Date: Fri, 29 Jul 2016 15:50:19 +0300 Subject: [PATCH 3/4] code reviewed [#MOBMET-3538] --- .../libdaemon/include/daemon/GraphiteWriter.h | 2 +- libs/libdaemon/src/GraphiteWriter.cpp | 22 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/libs/libdaemon/include/daemon/GraphiteWriter.h b/libs/libdaemon/include/daemon/GraphiteWriter.h index 6bbb17ece40..10f062479d6 100644 --- a/libs/libdaemon/include/daemon/GraphiteWriter.h +++ b/libs/libdaemon/include/daemon/GraphiteWriter.h @@ -37,7 +37,7 @@ public: /// путь вида prefix.environment.layer.daemon_name.metrica static std::string getPerLayerPath( const std::string & prefix = "one_min", - const boost::optional layer = {}); + const boost::optional & layer = {}); /// возвращает путь root_path.server_name static std::string getPerServerPath(const std::string & server_name, const std::string & root_path = "one_min"); diff --git a/libs/libdaemon/src/GraphiteWriter.cpp b/libs/libdaemon/src/GraphiteWriter.cpp index d65259ad98a..08358ffb268 100644 --- a/libs/libdaemon/src/GraphiteWriter.cpp +++ b/libs/libdaemon/src/GraphiteWriter.cpp @@ -75,19 +75,22 @@ std::string getEnvironment() } } -std::string getPostfix(const boost::optional layer) +std::string GraphiteWriter::getPerLayerPath( + const std::string & prefix, + const boost::optional & layer) { static const std::string environment = getEnvironment(); std::stringstream path_full; - path_full << environment << "."; + path_full << prefix << "." << environment << "."; const BaseDaemon & daemon = BaseDaemon::instance(); - if (layer) - path_full << "layer" << std::setfill('0') << std::setw(3) << *layer << "."; - else if (daemon.getLayer()) - path_full << "layer" << std::setfill('0') << std::setw(3) << *daemon.getLayer() << "."; + const boost::optional layer_ = layer.is_initialized() + ? layer + : daemon.getLayer(); + if (layer_) + path_full << "layer" << std::setfill('0') << std::setw(3) << *layer_ << "."; /// Когда несколько демонов запускается на одной машине /// к имени демона добавляется цифра. @@ -101,13 +104,6 @@ std::string getPostfix(const boost::optional layer) return path_full.str(); } -std::string GraphiteWriter::getPerLayerPath( - const std::string & prefix, - const boost::optional layer) -{ - return prefix + "." + getPostfix(layer); -} - std::string GraphiteWriter::getPerServerPath(const std::string & server_name, const std::string & root_path) { std::string path = root_path + "." + server_name; From 9c8697655ebcd23f56f6a71521c551ce985395d4 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 18 Aug 2016 14:17:07 +0300 Subject: [PATCH 4/4] build: fix build after merge with clickhouse repo --- dbms/CMakeLists.txt | 8 ++++++-- dbms/src/DataStreams/tests/json_streams.cpp | 2 +- libs/libzkutil/CMakeLists.txt | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index a93123ddd5d..3fe51b94a12 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -15,6 +15,11 @@ else() set (LINK_MONGOCLIENT libmongoclient.a libssl.a libcrypto.a libboost_thread.a) endif() + +add_library(string_utils + include/DB/Common/StringUtils.h + src/Common/StringUtils.cpp) + add_library (dbms src/Server/InterserverIOHTTPHandler.h src/Server/Server.h @@ -428,7 +433,6 @@ add_library (dbms include/DB/Common/getNumberOfPhysicalCPUCores.h include/DB/Common/BitHelpers.h include/DB/Common/BlockFilterCreator.h - include/DB/Common/StringUtils.h include/DB/Common/randomSeed.h include/DB/Common/unaligned.h include/DB/Common/ThreadPool.h @@ -601,7 +605,6 @@ add_library (dbms src/Common/ShellCommand.cpp src/Common/isLocalAddress.cpp src/Common/getNumberOfPhysicalCPUCores.cpp - src/Common/StringUtils.cpp src/Common/randomSeed.cpp src/Common/ThreadPool.cpp @@ -959,6 +962,7 @@ target_link_libraries(dbms mysqlxx cityhash farmhash metrohash lz4 zstd + string_utils double-conversion ${LINK_LIBRARIES_ONLY_ON_X86_64} re2 re2_st diff --git a/dbms/src/DataStreams/tests/json_streams.cpp b/dbms/src/DataStreams/tests/json_streams.cpp index e00e570ff05..fff7f38f0cc 100644 --- a/dbms/src/DataStreams/tests/json_streams.cpp +++ b/dbms/src/DataStreams/tests/json_streams.cpp @@ -62,7 +62,7 @@ try //copyData(row_input, row_output); BlockInputStreamFromRowInputStream in(std::make_shared(in_buf, sample, true, true), sample); - BlockOutputStreamFromRowOutputStream out(std::make_shared(out_buf, sample)); + BlockOutputStreamFromRowOutputStream out(std::make_shared(out_buf, sample, false)); copyData(in, out); } diff --git a/libs/libzkutil/CMakeLists.txt b/libs/libzkutil/CMakeLists.txt index a2ef154e649..4215235218b 100644 --- a/libs/libzkutil/CMakeLists.txt +++ b/libs/libzkutil/CMakeLists.txt @@ -16,6 +16,6 @@ add_library(zkutil include/zkutil/Types.h include/zkutil/ZooKeeperHolder.h) -target_link_libraries(zkutil zookeeper pthread PocoFoundation) +target_link_libraries(zkutil zookeeper pthread PocoFoundation string_utils) add_subdirectory (src)