Merge pull request #77 from yandex/metrika-sync

Metrika sync

dbms/CMakeLists.txt

@@ -15,6 +15,11 @@ else()
 	set (LINK_MONGOCLIENT libmongoclient.a libssl.a libcrypto.a libboost_thread.a)
 endif()
 
+
+add_library(string_utils
+	include/DB/Common/StringUtils.h
+	src/Common/StringUtils.cpp)
+
 add_library (dbms
 	src/Server/InterserverIOHTTPHandler.h
 	src/Server/Server.h
@@ -428,7 +433,6 @@ add_library (dbms
 	include/DB/Common/getNumberOfPhysicalCPUCores.h
 	include/DB/Common/BitHelpers.h
 	include/DB/Common/BlockFilterCreator.h
-	include/DB/Common/StringUtils.h
 	include/DB/Common/randomSeed.h
 	include/DB/Common/unaligned.h
 	include/DB/Common/ThreadPool.h
@@ -601,7 +605,6 @@ add_library (dbms
 	src/Common/ShellCommand.cpp
 	src/Common/isLocalAddress.cpp
 	src/Common/getNumberOfPhysicalCPUCores.cpp
-	src/Common/StringUtils.cpp
 	src/Common/randomSeed.cpp
 	src/Common/ThreadPool.cpp
 
@@ -959,6 +962,7 @@ target_link_libraries(dbms
 	mysqlxx
 	cityhash farmhash metrohash
 	lz4 zstd
+	string_utils
 	double-conversion
 	${LINK_LIBRARIES_ONLY_ON_X86_64}
 	re2 re2_st

dbms/src/DataStreams/tests/json_streams.cpp

@@ -62,7 +62,7 @@ try
 		//copyData(row_input, row_output);
 
 		BlockInputStreamFromRowInputStream in(std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, true, true), sample);
-		BlockOutputStreamFromRowOutputStream out(std::make_shared<JSONRowOutputStream>(out_buf, sample));
+		BlockOutputStreamFromRowOutputStream out(std::make_shared<JSONRowOutputStream>(out_buf, sample, false));
 		copyData(in, out);
 	}
 

libs/libdaemon/include/daemon/GraphiteWriter.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <boost/optional.hpp>
 #include <string>
 #include <time.h>
 #include <Poco/Net/StreamSocket.h>
@@ -34,7 +35,9 @@ public:
 
 	/// Для облачных демонов удобней использовать
 	/// путь вида prefix.environment.layer.daemon_name.metrica
-	static std::string getPerLayerPath(const std::string & prefix = "one_min");
+	static std::string getPerLayerPath(
+		const std::string & prefix = "one_min",
+		const boost::optional<std::size_t> & layer = {});
 	
 	/// возвращает путь root_path.server_name
 	static std::string getPerServerPath(const std::string & server_name, const std::string & root_path = "one_min");

libs/libdaemon/src/GraphiteWriter.cpp

@@ -49,32 +49,48 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin
 		root_path += "." + sub_path;
 }
 
-std::string getPostfix()
+/// Угадываем имя среды по имени машинки
+/// машинки имеют имена вида example01dt.yandex.ru
+/// t - test
+/// dev - development
+/// никакого суффикса - production
+std::string getEnvironment()
 {
-	/// Угадываем имя среды по имени машинки
-	/// машинки имеют имена вида example01dt.yandex.ru
-	/// t - test
-	/// dev - development
-	/// никакого суффикса - production
-
-	std::stringstream path_full;
-
 	std::string hostname = Poco::Net::DNS::hostName();
 	hostname = hostname.substr(0, hostname.find('.'));
 
 	const std::string development_suffix = "dev";
 	if (hostname.back() == 't')
-		path_full << "test.";
+	{
+		return "test";
+	}
 	else if (hostname.size() > development_suffix.size() &&
 			hostname.substr(hostname.size() - development_suffix.size()) == development_suffix)
-		path_full << "development.";
+	{
+		return "development";
+	}
 	else
-		path_full << "production.";
+	{
+		return "production";
+	}
+}
+
+std::string GraphiteWriter::getPerLayerPath(
+	const std::string & prefix,
+	const boost::optional<std::size_t> & layer)
+{
+	static const std::string environment = getEnvironment();
+
+	std::stringstream path_full;
+	path_full << prefix << "." << environment << ".";
 
 	const BaseDaemon & daemon = BaseDaemon::instance();
 
-	if (daemon.getLayer())
-		path_full << "layer" << std::setfill('0') << std::setw(3) << *daemon.getLayer() << ".";
+	const boost::optional<std::size_t> layer_ = layer.is_initialized()
+	    ? layer
+	    : daemon.getLayer();
+	if (layer_)
+		path_full << "layer" << std::setfill('0') << std::setw(3) << *layer_ << ".";
 
 	/// Когда несколько демонов запускается на одной машине
 	/// к имени демона добавляется цифра.
@@ -88,12 +104,6 @@ std::string getPostfix()
 	return path_full.str();
 }
 
-std::string GraphiteWriter::getPerLayerPath(const std::string & prefix)
-{
-	const std::string static postfix = getPostfix();
-	return prefix + "." + postfix;
-}
-
 std::string GraphiteWriter::getPerServerPath(const std::string & server_name, const std::string & root_path)
 {
 	std::string path = root_path + "." + server_name;

libs/libzkutil/CMakeLists.txt

@@ -16,6 +16,6 @@ add_library(zkutil
     include/zkutil/Types.h
     include/zkutil/ZooKeeperHolder.h)
 
-target_link_libraries(zkutil zookeeper pthread PocoFoundation)
+target_link_libraries(zkutil zookeeper pthread PocoFoundation string_utils)
 
 add_subdirectory (src)