Merge pull request #477 from yandex/merge_with_metrika

Merge with metrika
This commit is contained in:
alexey-milovidov 2017-02-11 00:32:34 +04:00 committed by GitHub
commit 9f243d1637
20 changed files with 155 additions and 247 deletions

View File

@ -1,7 +1,7 @@
project (ClickHouse)
cmake_minimum_required (VERSION 2.6)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/")
message(STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
@ -166,7 +166,11 @@ endif (TEST_COVERAGE)
# Run tests with "make check"
if (ENABLE_TESTS)
message (STATUS "Tests are enabled")
include (cmake/add_check.cmake)
if (TARGET check)
message (STATUS "Target check already exists")
else ()
include (cmake/add_check.cmake)
endif ()
endif (ENABLE_TESTS)
# Installation prefix

View File

@ -1,14 +1,14 @@
include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
include_directories (${ClickHouse_SOURCE_DIR}/dbms/include)
# TODO:
# move code with incldes from .h to .cpp and clean this list:
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libcommon/include)
# for generated config_version.h and config_common.h:
include_directories (${CMAKE_BINARY_DIR}/libs/libcommon/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libpocoext/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libzkutil/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libmysqlxx/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libzookeeper/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libcityhash/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libdouble-conversion)
include_directories (${ClickHouse_BINARY_DIR}/libs/libcommon/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libpocoext/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libzkutil/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libmysqlxx/include)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdouble-conversion)

View File

@ -1,4 +1,4 @@
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
@ -128,7 +128,8 @@ else()
set(PLATFORM_LIBS "")
endif()
include (${CMAKE_SOURCE_DIR}/cmake/find_iconv.cmake)
include (${ClickHouse_SOURCE_DIR}/cmake/find_iconv.cmake)
find_package (Threads)
target_link_libraries(dbms

View File

@ -25,6 +25,7 @@ public:
* то будет также загружен файл /opt/geo/regions_hierarchy_ua.txt, если такой есть - он будет доступен по ключу ua.
*/
RegionsHierarchies();
explicit RegionsHierarchies(const std::string & path_to_regions_hierarchy_file);
/// Has corresponding section in configuration file.
static bool isConfigured();

View File

@ -68,6 +68,7 @@ public:
/** Перезагружает, при необходимости, имена регионов.
*/
void reload();
void reload(const std::string & directory);
/// Has corresponding section in configuration file.
static bool isConfigured();

View File

@ -9,11 +9,15 @@ static constexpr auto config_key = "path_to_regions_hierarchy_file";
RegionsHierarchies::RegionsHierarchies()
: RegionsHierarchies(Poco::Util::Application::instance().config().getString(config_key))
{
}
RegionsHierarchies::RegionsHierarchies(const std::string & path)
{
Logger * log = &Logger::get("RegionsHierarchies");
std::string path = Poco::Util::Application::instance().config().getString(config_key);
LOG_DEBUG(log, "Adding default regions hierarchy from " << path);
data.emplace(std::piecewise_construct,

View File

@ -28,14 +28,17 @@ std::string RegionsNames::dumpSupportedLanguagesNames()
return res;
}
void RegionsNames::reload()
{
std::string directory = Poco::Util::Application::instance().config().getString(config_key);
reload(directory);
}
void RegionsNames::reload(const std::string & directory)
{
Logger * log = &Logger::get("RegionsNames");
LOG_DEBUG(log, "Reloading regions names");
std::string directory = Poco::Util::Application::instance().config().getString(config_key);
RegionID max_region_id = 0;
for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id)
{

View File

@ -3,9 +3,9 @@ set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/include/common/config_common.h)
set (CONFIG_BUILD ${CMAKE_CURRENT_BINARY_DIR}/src/config_build.cpp)
include_directories (include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libcctz/include)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcctz/include)
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
if (APPLE)
if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND NOT "${CMAKE_SYSTEM_VERSION}" VERSION_LESS "16.1.0")
@ -20,8 +20,8 @@ message (STATUS "Will build ${VERSION_FULL}")
configure_file (${CMAKE_CURRENT_SOURCE_DIR}/include/common/config_common.h.in ${CONFIG_COMMON})
configure_file (${CMAKE_CURRENT_SOURCE_DIR}/include/common/config_version.h.in ${CONFIG_VERSION})
get_property (BUILD_COMPILE_DEFINITIONS DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTY COMPILE_DEFINITIONS)
get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
get_property (BUILD_COMPILE_DEFINITIONS DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY COMPILE_DEFINITIONS)
get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
string (TIMESTAMP BUILD_DATE "%Y-%m-%d" UTC)
configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/config_build.cpp.in ${CONFIG_BUILD})

View File

@ -1,128 +0,0 @@
#pragma once
#include <stdexcept>
#include <sstream>
#include <iostream>
/// Часть функциональности - только для сборки из репозитория Метрики.
#ifndef NO_METRIKA
#include <ssqls/IDbObject.h>
#else
struct IDbObject {};
#endif
#include <regex>
#include <iostream>
#include <DB/Common/StackTrace.h>
#include <DB/IO/WriteBufferFromString.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/Logger.h>
#include <Poco/AutoPtr.h>
namespace Test
{
template <class Result, class Reference, class Enable = void>
struct Comparator
{
void check(const Result & result, const Reference & reference, const std::string & check_name = "")
{
if (result != reference)
{
std::stringstream ss;
if (check_name.size())
ss << "Check name: " << check_name << ". ";
StackTrace stacktrace;
ss << "Result ";
if (check_name.size())
ss << "for \"" << check_name << "\" ";
ss << "differs from reference" <<
". Result: " << result << ", reference: " << reference << "\n. Stacktrace: " << stacktrace.toString();
throw std::logic_error(ss.str());
}
}
};
template <class Result, class Reference, class Enable = void, class ... Args>
void compare(const Result & result, const Reference & reference, Args && ... args)
{
Comparator<Result, Reference> comp;
comp.check(result, reference, std::forward<Args>(args)...);
}
template <class ContainerA, class ContainerB>
void compareContainers(const ContainerA & result, const ContainerB & reference)
{
if (result != reference)
{
std::stringstream ss;
ss << "Result differs from reference. Result: {";
for (auto a : result)
ss << " " << a;
ss << " }, reference {";
for (auto b : reference)
ss << " " << b;
ss << " }";
StackTrace stacktrace;
ss << "\n. Stacktrace: " << stacktrace.toString();
throw std::logic_error(ss.str());
}
}
template <class Result, class Reference>
struct Comparator<Result, Reference, typename std::enable_if<
std::is_base_of<IDbObject, Result>::value &&
std::is_base_of<IDbObject, Reference>::value>::type>
{
void check(const Result & result, const Reference & reference, const std::string field_name_regexp_str = ".*")
{
std::string res_s;
{
DB::WriteBufferFromString res_b(res_s);
result.writeTSKV(res_b, "", "", "");
}
std::string ref_s;
{
DB::WriteBufferFromString ref_b(ref_s);
reference.writeTSKV(ref_b, "", "", "");
}
size_t ref_pos = 0;
size_t res_pos = 0;
while (ref_pos != std::string::npos && res_pos != std::string::npos)
{
size_t new_ref_pos = ref_s.find('\t', ref_pos);
size_t new_res_pos = res_s.find('\t', res_pos);
auto ref_field = ref_s.substr(ref_pos,
new_ref_pos != std::string::npos ? new_ref_pos - ref_pos : new_ref_pos);
if (std::regex_match(ref_field, std::regex(field_name_regexp_str + "=.*")))
{
auto res_field = res_s.substr(res_pos, new_res_pos != std::string::npos ? new_res_pos - res_pos : new_res_pos);
compare(res_field, ref_field);
}
res_pos = new_res_pos != res_s.size() && new_res_pos != std::string::npos ?
new_res_pos + 1 : std::string::npos;
ref_pos = new_ref_pos != ref_s.size() && new_ref_pos != std::string::npos ?
new_ref_pos + 1 : std::string::npos;
}
}
};
inline void initLogger()
{
Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cerr);
Poco::Logger::root().setChannel(channel);
Poco::Logger::root().setLevel("trace");
}
};

View File

@ -1,13 +1,16 @@
#define NO_METRIKA
#include <common/Test.h>
#define BOOST_TEST_MODULE JSON_MODULE
#include <vector>
#include <string>
#include <exception>
#include <common/JSON.h>
#include <boost/range/irange.hpp>
using namespace std::literals::string_literals;
#include <boost/test/included/unit_test.hpp>
enum class ResultType
{
Return,
@ -21,7 +24,7 @@ struct GetStringTestRecord
std::string result;
};
void get_string()
void test()
{
std::vector<GetStringTestRecord> test_data =
{
@ -641,20 +644,26 @@ void get_string()
try
{
JSON j(r.input.c_str(), r.input.c_str() + r.input.size());
Test::compare(j.getString(), r.result);
Test::compare(r.result_type == ResultType::Return, true);
BOOST_CHECK_EQUAL(j.getString(), r.result);
BOOST_CHECK(r.result_type == ResultType::Return);
}
catch (JSONException & e)
{
Test::compare(r.result_type == ResultType::Throw, true);
Test::compare(e.message(), r.result);
BOOST_CHECK(r.result_type == ResultType::Throw);
BOOST_CHECK_EQUAL(e.message(), r.result);
}
}
}
}
int main()
BOOST_AUTO_TEST_SUITE(JSON_Suite)
BOOST_AUTO_TEST_CASE(SimpleTest)
{
get_string();
return 0;
test();
}
BOOST_AUTO_TEST_SUITE_END()

View File

@ -1,5 +1,5 @@
include_directories (include)
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
add_library (daemon
src/BaseDaemon.cpp
@ -7,6 +7,7 @@ add_library (daemon
include/daemon/BaseDaemon.h
include/daemon/GraphiteWriter.h
include/daemon/OwnPatternFormatter.h
)
target_link_libraries (daemon dbms)

View File

@ -0,0 +1,83 @@
#pragma once
#include <Poco/PatternFormatter.h>
#include <Poco/Ext/ThreadNumber.h>
#include <DB/IO/WriteBufferFromString.h>
#include <DB/IO/WriteHelpers.h>
#include <daemon/BaseDaemon.h>
#include <experimental/optional>
#include <functional>
/** Форматирует по своему.
* Некоторые детали невозможно получить, используя только Poco::PatternFormatter.
*
* Во-первых, используется номер потока не среди потоков Poco::Thread,
* а среди всех потоков, для которых был получен номер (см. ThreadNumber.h)
*
* Во-вторых, корректно выводится локальная дата и время.
* Poco::PatternFormatter плохо работает с локальным временем,
* в ситуациях, когда в ближайшем будущем намечается отмена или введение daylight saving time.
* - см. исходники Poco и http://thread.gmane.org/gmane.comp.time.tz/8883
*
* Также сделан чуть более эффективным (что имеет мало значения).
*/
class OwnPatternFormatter : public Poco::PatternFormatter
{
public:
enum Options
{
ADD_NOTHING = 0,
ADD_LAYER_TAG = 1 << 0
};
OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING) : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {}
void format(const Poco::Message & msg, std::string & text) override
{
DB::WriteBufferFromString wb(text);
/// For syslog: tag must be before message and first whitespace.
if (options & ADD_LAYER_TAG && daemon)
{
auto layer = daemon->getLayer();
if (layer)
{
writeCString("layer[", wb);
DB::writeIntText(*layer, wb);
writeCString("]: ", wb);
}
}
/// Output time with microsecond resolution.
timeval tv;
if (0 != gettimeofday(&tv, nullptr))
DB::throwFromErrno("Cannot gettimeofday");
/// Change delimiters in date for compatibility with old logs.
DB::writeDateTimeText<'.', ':'>(tv.tv_sec, wb);
DB::writeChar('.', wb);
DB::writeChar('0' + ((tv.tv_usec / 100000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 10000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 1000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 100) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 10) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 1) % 10), wb);
writeCString(" [ ", wb);
DB::writeIntText(Poco::ThreadNumber::get(), wb);
writeCString(" ] <", wb);
DB::writeString(getPriorityName(static_cast<int>(msg.getPriority())), wb);
writeCString("> ", wb);
DB::writeString(msg.getSource(), wb);
writeCString(": ", wb);
DB::writeString(msg.getText(), wb);
}
private:
const BaseDaemon * daemon;
Options options;
};

View File

@ -56,6 +56,7 @@
#include <DB/IO/WriteHelpers.h>
#include <common/ClickHouseRevision.h>
#include <daemon/OwnPatternFormatter.h>
using Poco::Logger;
using Poco::AutoPtr;
@ -471,78 +472,6 @@ void BaseDaemon::reloadConfiguration()
}
/** Форматирует по своему.
* Некоторые детали невозможно получить, используя только Poco::PatternFormatter.
*
* Во-первых, используется номер потока не среди потоков Poco::Thread,
* а среди всех потоков, для которых был получен номер (см. ThreadNumber.h)
*
* Во-вторых, корректно выводится локальная дата и время.
* Poco::PatternFormatter плохо работает с локальным временем,
* в ситуациях, когда в ближайшем будущем намечается отмена или введение daylight saving time.
* - см. исходники Poco и http://thread.gmane.org/gmane.comp.time.tz/8883
*
* Также сделан чуть более эффективным (что имеет мало значения).
*/
class OwnPatternFormatter : public Poco::PatternFormatter
{
public:
enum Options
{
ADD_NOTHING = 0,
ADD_LAYER_TAG = 1 << 0
};
OwnPatternFormatter(const BaseDaemon & daemon_, Options options_ = ADD_NOTHING) : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {}
void format(const Message & msg, std::string & text) override
{
DB::WriteBufferFromString wb(text);
/// For syslog: tag must be before message and first whitespace.
if (options & ADD_LAYER_TAG)
{
auto layer = daemon.getLayer();
if (layer)
{
writeCString("layer[", wb);
DB::writeIntText(*layer, wb);
writeCString("]: ", wb);
}
}
/// Output time with microsecond resolution.
timeval tv;
if (0 != gettimeofday(&tv, nullptr))
DB::throwFromErrno("Cannot gettimeofday");
/// Change delimiters in date for compatibility with old logs.
DB::writeDateTimeText<'.', ':'>(tv.tv_sec, wb);
DB::writeChar('.', wb);
DB::writeChar('0' + ((tv.tv_usec / 100000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 10000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 1000) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 100) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 10) % 10), wb);
DB::writeChar('0' + ((tv.tv_usec / 1) % 10), wb);
writeCString(" [ ", wb);
DB::writeIntText(Poco::ThreadNumber::get(), wb);
writeCString(" ] <", wb);
DB::writeString(getPriorityName(static_cast<int>(msg.getPriority())), wb);
writeCString("> ", wb);
DB::writeString(msg.getSource(), wb);
writeCString(": ", wb);
DB::writeString(msg.getText(), wb);
}
private:
const BaseDaemon & daemon;
Options options;
};
/// Для создания и уничтожения unique_ptr, который в заголовочном файле объявлен от incomplete type.
BaseDaemon::BaseDaemon() = default;
@ -611,7 +540,7 @@ void BaseDaemon::buildLoggers()
Poco::AutoPtr<SplitterChannel> split = new SplitterChannel;
// set up two channel chains
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(*this);
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
pf->setProperty("times", "local");
Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
log_file = new FileChannel;
@ -629,7 +558,7 @@ void BaseDaemon::buildLoggers()
std::cerr << "Should error logs to " << config().getString("logger.errorlog") << std::endl;
Poco::AutoPtr<Poco::LevelFilterChannel> level = new Poco::LevelFilterChannel;
level->setLevel(Message::PRIO_NOTICE);
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(*this);
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
pf->setProperty("times", "local");
Poco::AutoPtr<FormattingChannel> errorlog = new FormattingChannel(pf);
error_log_file = new FileChannel;
@ -646,7 +575,7 @@ void BaseDaemon::buildLoggers()
if (config().getBool("logger.use_syslog", false) || config().getBool("dynamic_layer_selection", false))
{
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(*this, OwnPatternFormatter::ADD_LAYER_TAG);
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG);
pf->setProperty("times", "local");
Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
syslog_channel = new Poco::SyslogChannel(commandName(), Poco::SyslogChannel::SYSLOG_CONS | Poco::SyslogChannel::SYSLOG_PID, Poco::SyslogChannel::SYSLOG_DAEMON);
@ -663,7 +592,7 @@ void BaseDaemon::buildLoggers()
{
// Выводим на консоль
Poco::AutoPtr<ConsoleChannel> file = new ConsoleChannel;
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(*this);
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this);
pf->setProperty("times", "local");
Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
log->setChannel(file);

View File

@ -2,8 +2,8 @@
include (cmake/find_mysqlclient.cmake)
include_directories (include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libcommon/include)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
add_library (mysqlxx
src/Connection.cpp

View File

@ -1,5 +1,5 @@
include_directories (include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libcommon/include)
add_library (pocoext
src/LevelFilterChannel.cpp

View File

@ -1,7 +1,7 @@
include_directories (include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libcommon/include)
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
add_library(zkutil
src/ZooKeeper.cpp

View File

@ -1,4 +1,4 @@
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
if (NOT NO_WERROR)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")

View File

@ -1,5 +1,5 @@
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libzstd/include)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include)
add_executable (clickhouse-compressor main.cpp)
target_link_libraries (clickhouse-compressor dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})

View File

@ -1,4 +1,4 @@
include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
include_directories (${ClickHouse_SOURCE_DIR}/dbms/include)
add_executable(corrector_utf8 corrector_utf8.cpp)

View File

@ -1,5 +1,5 @@
include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
include_directories (${ClickHouse_SOURCE_DIR}/dbms/include)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libcommon/include)
add_executable (iotest iotest.cpp ${SRCS})
target_link_libraries (iotest dbms)