thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 09:02:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #80 from yandex/metrica-sync2

Browse Source 
Metrica sync2
This commit is contained in:
alexey-milovidov 2016-08-23 18:18:19 +04:00 committed by GitHub
commit 9148209bc2
18 changed files with 283 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -26,7 +26,11 @@ IF(NOT CMAKE_BUILD_TYPE)
ENDIF()
MESSAGE( STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} )
set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
# ASan - build type with address sanitizer
# UBSan - build type with undefined behaviour sanitizer
# TSan is not supported due to false positive errors in libstdc++ and necessity to rebuild libstdc++ with TSan
set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBSan" CACHE STRING "" FORCE)
IF (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
SET(AARCH64 1)
@ -55,11 +59,17 @@ SET(CMAKE_BUILD_COLOR_MAKEFILE ON)
SET(CMAKE_CXX_FLAGS "-std=gnu++1y ${COMMON_WARNING_FLAGS} -Wnon-virtual-dtor ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g")
SET(CMAKE_C_FLAGS "${COMMON_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g")
SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ ${GLIBC_COMPATIBILITY_LINK_FLAGS}")
SET(CMAKE_CXX_FLAGS_ASAN "-O3 -g -fsanitize=address -fno-omit-frame-pointer")
SET(CMAKE_CXX_FLAGS_UBSAN "-O3 -g -fsanitize=undefined -fno-omit-frame-pointer")
SET(CMAKE_C_FLAGS_ASAN "-O3 -g -fsanitize=address -fno-omit-frame-pointer")
SET(CMAKE_C_FLAGS_UBSAN "-O3 -g -fsanitize=undefined -fno-omit-frame-pointer")
# cmake -DCMAKE_BUILD_TYPE=Debug ..
SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")

4
contrib/libpoco/Foundation/include/Poco/Task.h
View File

@ -26,6 +26,8 @@
#include "Poco/Mutex.h"
#include "Poco/Event.h"
#include <atomic>
namespace Poco {
@ -140,7 +142,7 @@ private:
std::string _name;
TaskManager* _pOwner;
float _progress;
TaskState _state;
std::atomic<TaskState> _state;
Event _cancelEvent;
mutable FastMutex _mutex;

15
contrib/libpoco/Foundation/src/Task.cpp
View File

@ -17,7 +17,7 @@
#include "Poco/Task.h"
#include "Poco/TaskManager.h"
#include "Poco/Exception.h"
#include <array>
namespace Poco {
@ -61,8 +61,17 @@ void Task::run()
pOwner->taskStarted(this);
try
{
_state = TASK_RUNNING;
runTask();
/** Task can be already cancelled.
* To prevent endless executing already cancelled task _state is assigned to TASK_RUNNING only if _state != TASK_CANCELLING
*/
std::array<TaskState, 3> allowed_states{TASK_IDLE, TASK_STARTING, TASK_FINISHED};
for (auto & expected : allowed_states)
if (_state.compare_exchange_strong(expected, TASK_RUNNING))
break;
if (_state == TASK_RUNNING)
runTask();
}
catch (Exception& exc)
{

10
dbms/include/DB/Columns/ColumnVector.h
View File

@ -265,6 +265,16 @@ public:
res = typename NearestFieldType<T>::Type(data[n]);
}
const T & getElement(size_t n) const
{
return data[n];
}
T & getElement(size_t n)
{
return data[n];
}
UInt64 get64(size_t n) const override
{
return unionCastToUInt64(data[n]);

60
dbms/include/DB/Common/ConcurrentBoundedQueue.h
View File

@ -1,12 +1,42 @@
#pragma once
#include <queue>
#include <type_traits>
#include <Poco/Mutex.h>
#include <Poco/Semaphore.h>
#include <DB/Core/Types.h>
namespace detail
{
template <class T, bool is_nothrow_move_assignable = std::is_nothrow_move_assignable<T>::value>
struct MoveOrCopyIfThrow;
template <class T>
struct MoveOrCopyIfThrow<T, true>
{
void operator()(T && src, T & dst) const
{
dst = std::forward<T>(src);
}
};
template <class T>
struct MoveOrCopyIfThrow<T, false>
{
void operator()(T && src, T & dst) const
{
dst = src;
}
};
template <class T>
void moveOrCopyIfThrow(T && src, T & dst)
{
MoveOrCopyIfThrow<T>()(std::forward<T>(src), dst);
}
};
/** Очень простая thread-safe очередь ограниченной длины.
* Если пытаться вынуть элемент из пустой очереди, то поток блокируется, пока очередь не станет непустой.
@ -36,12 +66,23 @@ public:
fill_count.set();
}
template <class ... Args>
void emplace(Args && ... args)
{
empty_count.wait();
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.emplace(std::forward<Args>(args)...);
}
fill_count.set();
}
void pop(T & x)
{
fill_count.wait();
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
x = queue.front();
detail::moveOrCopyIfThrow(std::move(queue.front()), x);
queue.pop();
}
empty_count.set();
@ -61,13 +102,28 @@ public:
return false;
}
template <class ... Args>
bool tryEmplace(DB::UInt64 milliseconds, Args && ... args)
{
if (empty_count.tryWait(milliseconds))
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
queue.emplace(std::forward<Args>(args)...);
}
fill_count.set();
return true;
}
return false;
}
bool tryPop(T & x, DB::UInt64 milliseconds = 0)
{
if (fill_count.tryWait(milliseconds))
{
{
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
x = queue.front();
detail::moveOrCopyIfThrow(std::move(queue.front()), x);
queue.pop();
}
empty_count.set();

10
libs/libcommon/include/common/Common.h
View File

@ -43,13 +43,3 @@ using ClickID_t = UInt64;
/** Идентификатор цели */
using GoalID_t = UInt32;
namespace std
{
template<>
struct hash<VisitID_t> : public unary_function<VisitID_t, size_t>
{
size_t operator()(VisitID_t x) const { return x; }
};
}

13
libs/libcommon/include/common/DateLUTImpl.h
View File

@ -486,16 +486,3 @@ public:
return s;
}
};
namespace std
{
template<>
struct hash<DayNum_t>
{
size_t operator() (DayNum_t x) const
{
return x;
}
};
}

10
libs/libcommon/include/common/MultiVersion.h
View File

@ -47,6 +47,11 @@ public:
set(std::move(value));
}
MultiVersion(std::unique_ptr<T> && value)
{
set(std::move(value));
}
/// Получить текущую версию для использования. Возвращает shared_ptr, который определяет время жизни версии.
const Version get() const
{
@ -68,6 +73,11 @@ public:
set(Version(value));
}
void set(std::unique_ptr<T> && value)
{
set(Version(value.release()));
}
private:
Version current_version;
mutable std::mutex mutex;

1
libs/libcommon/include/common/Test.h
View File

@ -2,6 +2,7 @@
#include <stdexcept>
#include <sstream>
#include <iostream>
/// Часть функциональности - только для сборки из репозитория Метрики.
#ifndef NO_METRIKA

71
libs/libcommon/include/common/strong_typedef.h
View File

@ -1,23 +1,62 @@
#pragma once
#include <boost/operators.hpp>
#include <type_traits>
/** https://svn.boost.org/trac/boost/ticket/5182
*/
#define STRONG_TYPEDEF(T, D) \
struct D \
: boost::totally_ordered1< D \
, boost::totally_ordered2< D, T \
> > \
{ \
T t; \
explicit D(const T t_) : t(t_) {}; \
D(): t() {}; \
D(const D & t_) : t(t_.t){} \
D & operator=(const D & rhs) { t = rhs.t; return *this;} \
D & operator=(const T & rhs) { t = rhs; return *this;} \
operator const T & () const {return t; } \
operator T & () { return t; } \
bool operator==(const D & rhs) const { return t == rhs.t; } \
bool operator<(const D & rhs) const { return t < rhs.t; } \
template <class T, class Tag>
struct StrongTypedef
: boost::totally_ordered1< StrongTypedef<T, Tag>
, boost::totally_ordered2< StrongTypedef<T, Tag>, T> >
{
using Self = StrongTypedef<T, Tag>;
T t;
template <class Enable = typename std::is_copy_constructible<T>::type>
explicit StrongTypedef(const T & t_) : t(t_) {};
template <class Enable = typename std::is_move_constructible<T>::type>
explicit StrongTypedef(T && t_) : t(std::move(t_)) {};
template <class Enable = typename std::is_default_constructible<T>::type>
StrongTypedef(): t() {};
StrongTypedef(const Self &) = default;
StrongTypedef(Self &&) = default;
Self & operator=(const Self &) = default;
Self & operator=(Self &&) = default;
template <class Enable = typename std::is_copy_assignable<T>::type>
Self & operator=(const T & rhs) { t = rhs; return *this;}
template <class Enable = typename std::is_move_assignable<T>::type>
Self & operator=(T && rhs) { t = std::move(rhs); return *this;}
operator const T & () const {return t; }
operator T & () { return t; }
bool operator==(const Self & rhs) const { return t == rhs.t; }
bool operator<(const Self & rhs) const { return t < rhs.t; }
T & toUnderType() { return t; }
const T & toUnderType() const { return t; }
};
namespace std
{
template <class T, class Tag>
struct hash<StrongTypedef<T, Tag>>
{
size_t operator()(const StrongTypedef<T, Tag> & x) const
{
return std::hash<T>()(x.toUnderType());
}
};
}
#define STRONG_TYPEDEF(T, D) \
struct D ## Tag {}; \
using D = StrongTypedef<T, D ## Tag>; \

3
libs/libcommon/src/tests/CMakeLists.txt
View File

@ -6,6 +6,7 @@ add_executable (date_lut3 date_lut3.cpp)
add_executable (date_lut4 date_lut4.cpp)
add_executable (multi_version multi_version.cpp)
add_executable (json_test json_test.cpp)
add_executable (strong_typedef strong_typedef.cpp)
target_link_libraries (date_lut_init common libicui18n.a libicuuc.a libicudata.a dl)
target_link_libraries (date_lut2 common libicui18n.a libicuuc.a libicudata.a dl)
@ -13,5 +14,7 @@ target_link_libraries (date_lut3 common libicui18n.a libicuuc.a libicudata.a dl)
target_link_libraries (date_lut4 common libicui18n.a libicuuc.a libicudata.a dl)
target_link_libraries (multi_version dbms libboost_system.a rt)
target_link_libraries (json_test dbms libboost_system.a rt)
target_link_libraries (strong_typedef common)
add_check (json_test)
add_check (strong_typedef)

65
libs/libcommon/src/tests/strong_typedef.cpp Normal file
View File

@ -0,0 +1,65 @@
#define BOOST_TEST_MODULE StrongTypedef
#include <common/strong_typedef.h>
#include <set>
#include <unordered_set>
#include <memory>
#include <type_traits>
#include <boost/test/included/unit_test.hpp>
BOOST_AUTO_TEST_SUITE(StrongTypedefSuite)
BOOST_AUTO_TEST_CASE(TypedefsOfTheSameType)
{
/// check that strong typedefs of same type differ
STRONG_TYPEDEF(int, Int);
STRONG_TYPEDEF(int, AnotherInt);
BOOST_CHECK(!(std::is_same<Int, AnotherInt>::value));
}
BOOST_AUTO_TEST_CASE(Map)
{
STRONG_TYPEDEF(int, Int);
/// check that this code compiles
std::set<Int> int_set;
int_set.insert(Int(1));
std::unordered_set<Int> int_unorderd_set;
int_unorderd_set.insert(Int(2));
}
BOOST_AUTO_TEST_CASE(CopyAndMoveCtor)
{
STRONG_TYPEDEF(int, Int);
Int a(1);
Int b(2);
a = b;
BOOST_CHECK_EQUAL(a.t, 2);
STRONG_TYPEDEF(std::unique_ptr<int>, IntPtr);
{
IntPtr ptr;
ptr = IntPtr(std::make_unique<int>(3));
BOOST_CHECK_EQUAL(*ptr.t, 3);
}
{
IntPtr ptr(std::make_unique<int>(3));
BOOST_CHECK_EQUAL(*ptr.t, 3);
}
}
BOOST_AUTO_TEST_CASE(NoDefaultCtor)
{
struct NoDefaultCtor
{
NoDefaultCtor(int i) {}
};
STRONG_TYPEDEF(NoDefaultCtor, MyStruct);
MyStruct m(1);
}
BOOST_AUTO_TEST_SUITE_END()

9
libs/libdaemon/include/daemon/GraphiteWriter.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <boost/optional.hpp>
#include <string>
#include <time.h>
#include <Poco/Net/StreamSocket.h>
@ -8,6 +7,7 @@
#include <Poco/Util/Application.h>
#include <common/logger_useful.h>
/// пишет в Graphite данные в формате
/// path value timestamp\n
/// path может иметь любую вложенность. Директории разделяются с помощью "."
@ -20,7 +20,6 @@ public:
template <typename T> using KeyValuePair = std::pair<std::string, T>;
template <typename T> using KeyValueVector = std::vector<KeyValuePair<T>>;
/// TODO: Сделать custom_root_path = getPerLayerPath() по умолчанию
template <typename T> void write(const std::string & key, const T & value,
time_t timestamp = 0, const std::string & custom_root_path = "")
{
@ -32,12 +31,6 @@ public:
{
writeImpl(key_val_vec, timestamp, custom_root_path);
}
/// Для облачных демонов удобней использовать
/// путь вида prefix.environment.layer.daemon_name.metrica
static std::string getPerLayerPath(
const std::string & prefix = "one_min",
const boost::optional<std::size_t> & layer = {});
/// возвращает путь root_path.server_name
static std::string getPerServerPath(const std::string & server_name, const std::string & root_path = "one_min");

54
libs/libdaemon/src/GraphiteWriter.cpp
View File

@ -49,60 +49,6 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin
root_path += "." + sub_path;
}
/// Угадываем имя среды по имени машинки
/// машинки имеют имена вида example01dt.yandex.ru
/// t - test
/// dev - development
/// никакого суффикса - production
std::string getEnvironment()
{
std::string hostname = Poco::Net::DNS::hostName();
hostname = hostname.substr(0, hostname.find('.'));
const std::string development_suffix = "dev";
if (hostname.back() == 't')
{
return "test";
}
else if (hostname.size() > development_suffix.size() &&
hostname.substr(hostname.size() - development_suffix.size()) == development_suffix)
{
return "development";
}
else
{
return "production";
}
}
std::string GraphiteWriter::getPerLayerPath(
const std::string & prefix,
const boost::optional<std::size_t> & layer)
{
static const std::string environment = getEnvironment();
std::stringstream path_full;
path_full << prefix << "." << environment << ".";
const BaseDaemon & daemon = BaseDaemon::instance();
const boost::optional<std::size_t> layer_ = layer.is_initialized()
? layer
: daemon.getLayer();
if (layer_)
path_full << "layer" << std::setfill('0') << std::setw(3) << *layer_ << ".";
/// Когда несколько демонов запускается на одной машине
/// к имени демона добавляется цифра.
/// Удалим последнюю цифру
std::locale locale;
std::string command_name = daemon.commandName();
if (std::isdigit(command_name.back(), locale))
command_name = command_name.substr(0, command_name.size() - 1);
path_full << command_name;
return path_full.str();
}
std::string GraphiteWriter::getPerServerPath(const std::string & server_name, const std::string & root_path)
{

5
libs/libzkutil/include/zkutil/KeeperException.h
View File

@ -50,6 +50,11 @@ public:
return isUnrecoverable() || code == ZCONNECTIONLOSS || code == ZOPERATIONTIMEOUT;
}
bool isTemporaryError() const
{
return code == ZCONNECTIONLOSS || code == ZOPERATIONTIMEOUT;
}
const int32_t code;
private:

2
libs/libzkutil/include/zkutil/Lock.h
View File

@ -31,6 +31,7 @@ namespace zkutil
}
Lock(const Lock &) = delete;
Lock(Lock && lock) = default;
Lock & operator=(const Lock &) = delete;
~Lock()
@ -58,6 +59,7 @@ namespace zkutil
Status tryCheck() const;
void unlock();
void unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks);
bool tryLock();

24
libs/libzkutil/include/zkutil/Types.h
View File

@ -19,6 +19,8 @@ public:
Op() : data(new zoo_op_t) {}
virtual ~Op() {}
virtual std::string describe() = 0;
std::unique_ptr<zoo_op_t> data;
struct Remove;
@ -35,6 +37,8 @@ struct Op::Remove : public Op
zoo_delete_op_init(data.get(), path.c_str(), version);
}
std::string describe() override { return "command: remove, path: " + path; }
private:
std::string path;
};
@ -48,6 +52,13 @@ struct Op::Create : public Op
return created_path.data();
}
std::string describe() override
{
return "command: create"
", path: " + path +
", value: " + value;
}
private:
std::string path;
std::string value;
@ -61,6 +72,16 @@ struct Op::SetData : public Op
{
zoo_set_op_init(data.get(), path.c_str(), value.c_str(), value.size(), version, &stat);
}
std::string describe() override
{
return
"command: set"
", path: " + path +
", value: " + value +
", version: " + std::to_string(data->set_op.version);
}
private:
std::string path;
std::string value;
@ -74,6 +95,9 @@ struct Op::Check : public Op
{
zoo_check_op_init(data.get(), path.c_str(), version);
}
std::string describe() override { return "command: check, path: " + path; }
private:
std::string path;
};

25
libs/libzkutil/src/Lock.cpp
View File

@ -16,6 +16,9 @@ bool Lock::tryLock()
{
size_t attempt;
std::string dummy;
/// TODO: ошибка. можно создать эфемерную ноду, но при этом не получить подтверждения даже после нескольких попыток.
/// тогда все последующие локи будут неуспешные из-за существования ноды.
int32_t code = zookeeper->tryCreateWithRetries(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy, &attempt);
if (code == ZNODEEXISTS)
@ -46,16 +49,16 @@ bool Lock::tryLock()
void Lock::unlock()
{
auto zookeeper = zookeeper_holder->getZooKeeper();
if (locked)
{
auto zookeeper = zookeeper_holder->getZooKeeper();
try
{
if (tryCheck() == Status::LOCKED_BY_ME)
{
size_t attempt;
int32_t code = zookeeper->tryRemoveEphemeralNodeWithRetries(lock_path, -1, &attempt);
if (attempt)
{
if (code != ZOK)
@ -117,3 +120,21 @@ std::string Lock::status2String(Status status)
return names[status];
}
void Lock::unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks)
{
try
{
unlock();
}
catch (const zkutil::KeeperException & e)
{
if (e.isTemporaryError())
{
LOG_WARNING(log, "Fail to unlock lock. Move lock to vector to remove later. Path: " << getPath());
failed_to_unlock_locks.emplace_back(std::move(*this));
}
else
throw;
}
}