thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 11:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into METR-19530

Browse Source 
Conflicts:
	metrica/src/CMakeLists.txt
	metrica/src/ClickHouse/libs/libdaemon/include/daemon/GraphiteWriter.h
	metrica/src/ClickHouse/libs/libdaemon/src/GraphiteWriter.cpp
	metrica/src/libs/CMakeLists.txt
	metrica/src/libs/liblogbroker-client/src/LogBroker.cpp
	metrica/src/libs/liblogbroker-log-import/include/logbroker-log-import/LogBrokerLogImportTask-inl.h
	metrica/src/libs/libstatdaemons/include/statdaemons/write_destinations/ClickHouseCluster.h
	metrica/src/programs/cleanerd/src/config.xml
	metrica/src/programs/deduplicatord/src/Task.cpp
	metrica/src/programs/visit-log-processor/src/VisitLogHandler.h
	metrica/src/tools/ssqls/templates/header.tpl
This commit is contained in:
Pavel Kartavyy 2016-08-19 17:15:58 +03:00
commit 6e7bb0cd65
571 changed files with 27784 additions and 14073 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.gitignore vendored
View File

@ -154,6 +154,34 @@ libs/libzkutil/src/tests/zkutil_zookeeper_holder
utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download-part
utils/zookeeper-dump-tree/zookeeper-dump-tree
utils/zookeeper-remove-by-list/zookeeper-remove-by-list
dbms/src/Storages/tests/remove_symlink_directory
dbms/tests/queries/1_stateful
debian/clickhouse-benchmark-metrika-yandex.postinst
debian/clickhouse-benchmark.docs
debian/clickhouse-client-metrika-yandex.postinst
debian/clickhouse-client.docs
debian/clickhouse-server-base.docs
debian/clickhouse-server-common.docs
debian/clickhouse-server-metrika-yandex.postinst
debian/clickhouse-server-metrika.docs
debian/clickhouse-server.docs
debian/compressor-metrika-yandex.postinst
debian/compressor.docs
debian/config-processor-metrika-yandex.postinst
debian/config-processor.docs
debian/control
debian/copyright
debian/tmp/
libs/libcommon/src/tests/json_test
utils/compressor/zstd_test
utils/wikistat-loader/wikistat-loader
dbms/src/Common/tests/pod_array
debian/clickhouse-benchmark/
debian/clickhouse-client/
debian/clickhouse-server-base/
debian/clickhouse-server-common/
debian/clickhouse-server-metrika/
debian/files
dbms/src/Server/data/*
dbms/src/Server/metadata/*
@ -195,3 +223,6 @@ config-preprocessed.xml
# Protobuf
*.pb.cpp
*.pb.h
# Ignore symlink to private repository
/private

3
.gitmodules vendored
View File

@ -1,3 +0,0 @@
[submodule "private"]
path = private
url = git@github.yandex-team.ru:Metrika/ClickHouse_private.git

39
CMakeLists.txt
View File

@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 2.6)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
# require at least gcc 5
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)
message(FATAL_ERROR "GCC version must be at least 5!")
message(FATAL_ERROR "GCC version must be at least 5! For example, if GCC 5 is available under gcc-5, g++-5 names, do the following: export CC=gcc-5 CXX=g++-5; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
endif()
elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
# require at least clang 3.8
@ -36,14 +36,29 @@ IF (NOT AARCH64)
SET(MACHINE_FLAGS "-msse4 -mpopcnt")
ENDIF()
SET(COMMON_WARNING_FLAGS "-Wall -Werror")
set (GLIBC_COMPATIBILITY FALSE CACHE BOOL "Set to TRUE to enable compatibility with older glibc libraries")
if ($ENV{GLIBC_COMPATIBILITY})
set (GLIBC_COMPATIBILITY TRUE)
endif()
if (GLIBC_COMPATIBILITY)
SET(GLIBC_COMPATIBILITY_COMPILE_FLAGS "-include ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/common/glibc_compatibility.h")
SET(GLIBC_COMPATIBILITY_LINK_FLAGS "-Wl,--wrap=memcpy")
endif()
SET(CMAKE_BUILD_COLOR_MAKEFILE ON)
SET(CMAKE_CXX_FLAGS "-std=gnu++1y -Wall -Werror -Wnon-virtual-dtor ${MACHINE_FLAGS}")
SET(CMAKE_CXX_FLAGS "-std=gnu++1y ${COMMON_WARNING_FLAGS} -Wnon-virtual-dtor ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g")
SET(CMAKE_C_FLAGS "-Wall -Werror ${MACHINE_FLAGS}")
SET(CMAKE_C_FLAGS "${COMMON_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
SET(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g")
SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ ${GLIBC_COMPATIBILITY_LINK_FLAGS}")
# cmake -DCMAKE_BUILD_TYPE=Debug ..
SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
@ -68,9 +83,6 @@ ENDIF(TESTS)
# Префикс для установки
SET(CMAKE_INSTALL_PREFIX /usr)
# Директория для файлов, специфичных для Яндекса
SET(CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libdivide/)
@ -82,7 +94,7 @@ include_directories (${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libsparsehash/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libre2/)
include_directories (${ClickHouse_BINARY_DIR}/contrib/libre2/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libboost-threadpool/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libpoco/Foundation/include/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libpoco/Util/include/)
include_directories (${ClickHouse_SOURCE_DIR}/contrib/libpoco/Net/include/)
@ -120,8 +132,17 @@ include_directories (/usr/local/include/)
link_directories (/usr/local/lib)
# Directory for Yandex specific files
SET(CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)
add_subdirectory (contrib)
add_subdirectory (libs)
add_subdirectory (utils)
add_subdirectory (dbms)
add_subdirectory (private)
IF (EXISTS ${CLICKHOUSE_PRIVATE_DIR})
add_subdirectory (private)
ENDIF()
message(STATUS "C_FLAGS: =${CMAKE_C_FLAGS}")
message(STATUS "CXX_FLAGS:=${CMAKE_CXX_FLAGS}")

2
contrib/CMakeLists.txt vendored
View File

@ -6,8 +6,8 @@ add_subdirectory (libfarmhash)
add_subdirectory (libmetrohash)
add_subdirectory (libpoco)
add_subdirectory (libre2)
add_subdirectory (libboost-threadpool)
add_subdirectory (libtcmalloc)
add_subdirectory (libzookeeper)
IF (NOT AARCH64)
add_subdirectory (libcpuid)

5
contrib/libboost-threadpool/COPYING
View File

@ -1,5 +0,0 @@
Copyright (c) 2005-2007 Philipp Henkel
Use, modification, and distribution are subject to the
Boost Software License, Version 1.0. (See accompanying file
LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

23
contrib/libboost-threadpool/LICENSE_1_0.txt
View File

@ -1,23 +0,0 @@
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

28
contrib/libboost-threadpool/threadpool.hpp
View File

@ -1,28 +0,0 @@
/*! \file
* \brief Main include.
*
* This is the only file you have to include in order to use the
* complete threadpool library.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_HPP_INCLUDED
#define THREADPOOL_HPP_INCLUDED
#include "threadpool/future.hpp"
#include "threadpool/pool.hpp"
#include "threadpool/pool_adaptors.hpp"
#include "threadpool/task_adaptors.hpp"
#endif // THREADPOOL_HPP_INCLUDED

215
contrib/libboost-threadpool/threadpool/detail/future.hpp
View File

@ -1,215 +0,0 @@
/*! \file
* \brief TODO.
*
* TODO.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_DETAIL_FUTURE_IMPL_HPP_INCLUDED
#define THREADPOOL_DETAIL_FUTURE_IMPL_HPP_INCLUDED
#include "locking_ptr.hpp"
#include <boost/smart_ptr.hpp>
#include <boost/optional.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/thread/xtime.hpp>
#include <boost/utility/result_of.hpp>
#include <boost/static_assert.hpp>
#include <boost/type_traits.hpp>
namespace boost { namespace threadpool { namespace detail
{
template<class Result>
class future_impl
{
public:
typedef Result const & result_type; //!< Indicates the functor's result type.
typedef Result future_result_type; //!< Indicates the future's result type.
typedef future_impl<future_result_type> future_type;
private:
volatile bool m_ready;
volatile future_result_type m_result;
mutable mutex m_monitor;
mutable condition m_condition_ready;
volatile bool m_is_cancelled;
volatile bool m_executing;
public:
public:
future_impl()
: m_ready(false)
, m_is_cancelled(false)
{
}
bool ready() const volatile
{
return m_ready;
}
void wait() const volatile
{
const future_type* self = const_cast<const future_type*>(this);
mutex::scoped_lock lock(self->m_monitor);
while(!m_ready)
{
self->m_condition_ready.wait(lock);
}
}
bool timed_wait(boost::xtime const & timestamp) const
{
const future_type* self = const_cast<const future_type*>(this);
mutex::scoped_lock lock(self->m_monitor);
while(!m_ready)
{
if(!self->m_condition_ready.timed_wait(lock, timestamp)) return false;
}
return true;
}
result_type operator()() const volatile
{
wait();
/*
if( throw_exception_ != 0 )
{
throw_exception_( this );
}
*/
return *(const_cast<const future_result_type*>(&m_result));
}
void set_value(future_result_type const & r) volatile
{
locking_ptr<future_type, mutex> lockedThis(*this, m_monitor);
if(!m_ready && !m_is_cancelled)
{
lockedThis->m_result = r;
lockedThis->m_ready = true;
lockedThis->m_condition_ready.notify_all();
}
}
/*
template<class E> void set_exception() // throw()
{
m_impl->template set_exception<E>();
}
template<class E> void set_exception( char const * what ) // throw()
{
m_impl->template set_exception<E>( what );
}
*/
bool cancel() volatile
{
if(!m_ready || m_executing)
{
m_is_cancelled = true;
return true;
}
else
{
return false;
}
}
bool is_cancelled() const volatile
{
return m_is_cancelled;
}
void set_execution_status(bool executing) volatile
{
m_executing = executing;
}
};
template<
template <typename> class Future,
typename Function
>
class future_impl_task_func
{
public:
typedef void result_type; //!< Indicates the functor's result type.
typedef Function function_type; //!< Indicates the function's type.
typedef typename result_of<function_type()>::type future_result_type; //!< Indicates the future's result type.
typedef Future<future_result_type> future_type; //!< Indicates the future's type.
// The task is required to be a nullary function.
BOOST_STATIC_ASSERT(function_traits<function_type()>::arity == 0);
// The task function's result type is required not to be void.
BOOST_STATIC_ASSERT(!is_void<future_result_type>::value);
private:
function_type m_function;
shared_ptr<future_type> m_future;
public:
future_impl_task_func(function_type const & function, shared_ptr<future_type> const & future)
: m_function(function)
, m_future(future)
{
}
void operator()()
{
if(m_function)
{
m_future->set_execution_status(true);
if(!m_future->is_cancelled())
{
// TODO future exeception handling
m_future->set_value(m_function());
}
m_future->set_execution_status(false); // TODO consider exceptions
}
}
};
} } } // namespace boost::threadpool::detail
#endif // THREADPOOL_DETAIL_FUTURE_IMPL_HPP_INCLUDED

102
contrib/libboost-threadpool/threadpool/detail/locking_ptr.hpp
View File

@ -1,102 +0,0 @@
/*! \file
* \brief The locking_ptr is smart pointer with a scoped locking mechanism.
*
* The class is a wrapper for a volatile pointer. It enables synchronized access to the
* internal pointer by locking the passed mutex.
* locking_ptr is based on Andrei Alexandrescu's LockingPtr. For more information
* see article "volatile - Multithreaded Programmer's Best Friend" by A. Alexandrescu.
*
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_DETAIL_LOCKING_PTR_HPP_INCLUDED
#define THREADPOOL_DETAIL_LOCKING_PTR_HPP_INCLUDED
#include <boost/utility.hpp>
// Support for old boost::thread
//**********************************************
#include <boost/thread/mutex.hpp>
#ifndef BOOST_THREAD_MUTEX_HPP
#include <boost/thread/detail/lock.hpp>
#endif
//**********************************************
namespace boost { namespace threadpool { namespace detail
{
/*! \brief Smart pointer with a scoped locking mechanism.
*
* This class is a wrapper for a volatile pointer. It enables synchronized access to the
* internal pointer by locking the passed mutex.
*/
template <typename T, typename Mutex>
class locking_ptr
: private noncopyable
{
T* m_obj; //!< The instance pointer.
Mutex & m_mutex; //!< Mutex is used for scoped locking.
public:
/// Constructor.
locking_ptr(volatile T& obj, const volatile Mutex& mtx)
: m_obj(const_cast<T*>(&obj))
, m_mutex(*const_cast<Mutex*>(&mtx))
{
// Lock mutex
#ifndef BOOST_THREAD_MUTEX_HPP
// Support for old boost::thread
boost::detail::thread::lock_ops<Mutex>::lock(m_mutex);
#else
m_mutex.lock();
#endif
}
/// Destructor.
~locking_ptr()
{
// Unlock mutex
#ifndef BOOST_THREAD_MUTEX_HPP
// Support for old boost::thread
boost::detail::thread::lock_ops<Mutex>::unlock(m_mutex);
#else
m_mutex.unlock();
#endif
}
/*! Returns a reference to the stored instance.
* \return The instance's reference.
*/
T& operator*() const
{
return *m_obj;
}
/*! Returns a pointer to the stored instance.
* \return The instance's pointer.
*/
T* operator->() const
{
return m_obj;
}
};
} } } // namespace boost::threadpool::detail
#endif // THREADPOOL_DETAIL_LOCKING_PTR_HPP_INCLUDED

453
contrib/libboost-threadpool/threadpool/detail/pool_core.hpp
View File

@ -1,453 +0,0 @@
/*! \file
* \brief Thread pool core.
*
* This file contains the threadpool's core class: pool<Task, SchedulingPolicy>.
*
* Thread pools are a mechanism for asynchronous and parallel processing
* within the same process. The pool class provides a convenient way
* for dispatching asynchronous tasks as functions objects. The scheduling
* of these tasks can be easily controlled by using customized schedulers.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_POOL_CORE_HPP_INCLUDED
#define THREADPOOL_POOL_CORE_HPP_INCLUDED
#include "locking_ptr.hpp"
#include "worker_thread.hpp"
#include "../task_adaptors.hpp"
#include <boost/thread.hpp>
#include <boost/thread/exceptions.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/smart_ptr.hpp>
#include <boost/bind.hpp>
#include <boost/static_assert.hpp>
#include <boost/type_traits.hpp>
#include <vector>
/// The namespace threadpool contains a thread pool and related utility classes.
namespace boost { namespace threadpool { namespace detail
{
/*! \brief Thread pool.
*
* Thread pools are a mechanism for asynchronous and parallel processing
* within the same process. The pool class provides a convenient way
* for dispatching asynchronous tasks as functions objects. The scheduling
* of these tasks can be easily controlled by using customized schedulers.
* A task must not throw an exception.
*
* A pool_impl is DefaultConstructible and NonCopyable.
*
* \param Task A function object which implements the operator 'void operator() (void) const'. The operator () is called by the pool to execute the task. Exceptions are ignored.
* \param Scheduler A task container which determines how tasks are scheduled. It is guaranteed that this container is accessed only by one thread at a time. The scheduler shall not throw exceptions.
*
* \remarks The pool class is thread-safe.
*
* \see Tasks: task_func, prio_task_func
* \see Scheduling policies: fifo_scheduler, lifo_scheduler, prio_scheduler
*/
template <
typename Task,
template <typename> class SchedulingPolicy,
template <typename> class SizePolicy,
template <typename> class SizePolicyController,
template <typename> class ShutdownPolicy
>
class pool_core
: public enable_shared_from_this< pool_core<Task, SchedulingPolicy, SizePolicy, SizePolicyController, ShutdownPolicy > >
, private noncopyable
{
public: // Type definitions
typedef Task task_type; //!< Indicates the task's type.
typedef SchedulingPolicy<task_type> scheduler_type; //!< Indicates the scheduler's type.
typedef pool_core<Task,
SchedulingPolicy,
SizePolicy,
SizePolicyController,
ShutdownPolicy > pool_type; //!< Indicates the thread pool's type.
typedef SizePolicy<pool_type> size_policy_type; //!< Indicates the sizer's type.
//typedef typename size_policy_type::size_controller size_controller_type;
typedef SizePolicyController<pool_type> size_controller_type;
// typedef SizePolicy<pool_type>::size_controller size_controller_type;
typedef ShutdownPolicy<pool_type> shutdown_policy_type;//!< Indicates the shutdown policy's type.
typedef worker_thread<pool_type> worker_type;
// The task is required to be a nullary function.
BOOST_STATIC_ASSERT(function_traits<task_type()>::arity == 0);
// The task function's result type is required to be void.
BOOST_STATIC_ASSERT(is_void<typename result_of<task_type()>::type >::value);
private: // Friends
friend class worker_thread<pool_type>;
#if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x580) // Tested with CC: Sun C++ 5.8 Patch 121018-08 2006/12/06
friend class SizePolicy;
friend class ShutdownPolicy;
#else
friend class SizePolicy<pool_type>;
friend class ShutdownPolicy<pool_type>;
#endif
private: // The following members may be accessed by _multiple_ threads at the same time:
volatile size_t m_worker_count;
volatile size_t m_target_worker_count;
volatile size_t m_active_worker_count;
private: // The following members are accessed only by _one_ thread at the same time:
scheduler_type m_scheduler;
scoped_ptr<size_policy_type> m_size_policy; // is never null
bool m_terminate_all_workers; // Indicates if termination of all workers was triggered.
std::vector<shared_ptr<worker_type> > m_terminated_workers; // List of workers which are terminated but not fully destructed.
private: // The following members are implemented thread-safe:
mutable recursive_mutex m_monitor;
mutable condition m_worker_idle_or_terminated_event; // A worker is idle or was terminated.
mutable condition m_task_or_terminate_workers_event; // Task is available OR total worker count should be reduced.
public:
/// Constructor.
pool_core()
: m_worker_count(0)
, m_target_worker_count(0)
, m_active_worker_count(0)
, m_terminate_all_workers(false)
{
pool_type volatile & self_ref = *this;
m_size_policy.reset(new size_policy_type(self_ref));
m_scheduler.clear();
}
/// Destructor.
~pool_core()
{
}
/*! Gets the size controller which manages the number of threads in the pool.
* \return The size controller.
* \see SizePolicy
*/
size_controller_type size_controller()
{
return size_controller_type(*m_size_policy, this->shared_from_this());
}
/*! Gets the number of threads in the pool.
* \return The number of threads.
*/
size_t size() const volatile
{
return m_worker_count;
}
// TODO is only called once
void shutdown()
{
ShutdownPolicy<pool_type>::shutdown(*this);
}
/*! Schedules a task for asynchronous execution. The task will be executed once only.
* \param task The task function object. It should not throw execeptions.
* \return true, if the task could be scheduled and false otherwise.
*/
bool schedule(task_type const & task) volatile
{
locking_ptr<pool_type, recursive_mutex> lockedThis(*this, m_monitor);
if(lockedThis->m_scheduler.push(task))
{
lockedThis->m_task_or_terminate_workers_event.notify_one();
return true;
}
else
{
return false;
}
}
/*! Returns the number of tasks which are currently executed.
* \return The number of active tasks.
*/
size_t active() const volatile
{
return m_active_worker_count;
}
/*! Returns the number of tasks which are ready for execution.
* \return The number of pending tasks.
*/
size_t pending() const volatile
{
locking_ptr<const pool_type, recursive_mutex> lockedThis(*this, m_monitor);
return lockedThis->m_scheduler.size();
}
/*! Removes all pending tasks from the pool's scheduler.
*/
void clear() volatile
{
locking_ptr<pool_type, recursive_mutex> lockedThis(*this, m_monitor);
lockedThis->m_scheduler.clear();
}
/*! Indicates that there are no tasks pending.
* \return true if there are no tasks ready for execution.
* \remarks This function is more efficient that the check 'pending() == 0'.
*/
bool empty() const volatile
{
locking_ptr<const pool_type, recursive_mutex> lockedThis(*this, m_monitor);
return lockedThis->m_scheduler.empty();
}
/*! The current thread of execution is blocked until the sum of all active
* and pending tasks is equal or less than a given threshold.
* \param task_threshold The maximum number of tasks in pool and scheduler.
*/
void wait(size_t const task_threshold = 0) const volatile
{
const pool_type* self = const_cast<const pool_type*>(this);
recursive_mutex::scoped_lock lock(self->m_monitor);
if(0 == task_threshold)
{
while(0 != self->m_active_worker_count || !self->m_scheduler.empty())
{
self->m_worker_idle_or_terminated_event.wait(lock);
}
}
else
{
while(task_threshold < self->m_active_worker_count + self->m_scheduler.size())
{
self->m_worker_idle_or_terminated_event.wait(lock);
}
}
}
/*! The current thread of execution is blocked until the timestamp is met
* or the sum of all active and pending tasks is equal or less
* than a given threshold.
* \param timestamp The time when function returns at the latest.
* \param task_threshold The maximum number of tasks in pool and scheduler.
* \return true if the task sum is equal or less than the threshold, false otherwise.
*/
bool wait(xtime const & timestamp, size_t const task_threshold = 0) const volatile
{
const pool_type* self = const_cast<const pool_type*>(this);
recursive_mutex::scoped_lock lock(self->m_monitor);
if(0 == task_threshold)
{
while(0 != self->m_active_worker_count || !self->m_scheduler.empty())
{
if(!self->m_worker_idle_or_terminated_event.timed_wait(lock, timestamp)) return false;
}
}
else
{
while(task_threshold < self->m_active_worker_count + self->m_scheduler.size())
{
if(!self->m_worker_idle_or_terminated_event.timed_wait(lock, timestamp)) return false;
}
}
return true;
}
private:
void terminate_all_workers(bool const wait) volatile
{
pool_type* self = const_cast<pool_type*>(this);
recursive_mutex::scoped_lock lock(self->m_monitor);
self->m_terminate_all_workers = true;
m_target_worker_count = 0;
self->m_task_or_terminate_workers_event.notify_all();
if(wait)
{
while(m_worker_count > 0)
{
self->m_worker_idle_or_terminated_event.wait(lock);
}
for(typename std::vector<shared_ptr<worker_type> >::iterator it = self->m_terminated_workers.begin();
it != self->m_terminated_workers.end();
++it)
{
(*it)->join();
}
self->m_terminated_workers.clear();
}
}
/*! Changes the number of worker threads in the pool. The resizing
* is handled by the SizePolicy.
* \param threads The new number of worker threads.
* \return true, if pool will be resized and false if not.
*/
bool resize(size_t const worker_count) volatile
{
locking_ptr<pool_type, recursive_mutex> lockedThis(*this, m_monitor);
if(!m_terminate_all_workers)
{
m_target_worker_count = worker_count;
}
else
{
return false;
}
if(m_worker_count <= m_target_worker_count)
{ // increase worker count
while(m_worker_count < m_target_worker_count)
{
try
{
worker_thread<pool_type>::create_and_attach(lockedThis->shared_from_this());
m_worker_count++;
m_active_worker_count++;
}
catch(thread_resource_error)
{
return false;
}
}
}
else
{ // decrease worker count
lockedThis->m_task_or_terminate_workers_event.notify_all(); // TODO: Optimize number of notified workers
}
return true;
}
// worker died with unhandled exception
void worker_died_unexpectedly(shared_ptr<worker_type> worker) volatile
{
locking_ptr<pool_type, recursive_mutex> lockedThis(*this, m_monitor);
m_worker_count--;
m_active_worker_count--;
lockedThis->m_worker_idle_or_terminated_event.notify_all();
if(m_terminate_all_workers)
{
lockedThis->m_terminated_workers.push_back(worker);
}
else
{
lockedThis->m_size_policy->worker_died_unexpectedly(m_worker_count);
}
}
void worker_destructed(shared_ptr<worker_type> worker) volatile
{
locking_ptr<pool_type, recursive_mutex> lockedThis(*this, m_monitor);
m_worker_count--;
m_active_worker_count--;
lockedThis->m_worker_idle_or_terminated_event.notify_all();
if(m_terminate_all_workers)
{
lockedThis->m_terminated_workers.push_back(worker);
}
}
bool execute_task() volatile
{
function0<void> task;
{ // fetch task
pool_type* lockedThis = const_cast<pool_type*>(this);
recursive_mutex::scoped_lock lock(lockedThis->m_monitor);
// decrease number of threads if necessary
if(m_worker_count > m_target_worker_count)
{
return false; // terminate worker
}
// wait for tasks
while(lockedThis->m_scheduler.empty())
{
// decrease number of workers if necessary
if(m_worker_count > m_target_worker_count)
{
return false; // terminate worker
}
else
{
m_active_worker_count--;
lockedThis->m_worker_idle_or_terminated_event.notify_all();
lockedThis->m_task_or_terminate_workers_event.wait(lock);
m_active_worker_count++;
}
}
task = lockedThis->m_scheduler.top();
lockedThis->m_scheduler.pop();
}
// call task function
if(task)
{
task();
}
//guard->disable();
return true;
}
};
} } } // namespace boost::threadpool::detail
#endif // THREADPOOL_POOL_CORE_HPP_INCLUDED

65
contrib/libboost-threadpool/threadpool/detail/scope_guard.hpp
View File

@ -1,65 +0,0 @@
/*! \file
* \brief TODO.
*
* TODO.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_DETAIL_SCOPE_GUARD_HPP_INCLUDED
#define THREADPOOL_DETAIL_SCOPE_GUARD_HPP_INCLUDED
#include <boost/function.hpp>
namespace boost { namespace threadpool { namespace detail
{
// TODO documentation
class scope_guard
: private boost::noncopyable
{
function0<void> const m_function;
bool m_is_active;
public:
scope_guard(function0<void> const & call_on_exit)
: m_function(call_on_exit)
, m_is_active(true)
{
}
~scope_guard()
{
if(m_is_active && m_function)
{
m_function();
}
}
void disable()
{
m_is_active = false;
}
};
} } } // namespace boost::threadpool::detail
#endif // THREADPOOL_DETAIL_SCOPE_GUARD_HPP_INCLUDED

115
contrib/libboost-threadpool/threadpool/detail/worker_thread.hpp
View File

@ -1,115 +0,0 @@
/*! \file
* \brief Thread pool worker.
*
* The worker thread instance is attached to a pool
* and executes tasks of this pool.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_DETAIL_WORKER_THREAD_HPP_INCLUDED
#define THREADPOOL_DETAIL_WORKER_THREAD_HPP_INCLUDED
#include "scope_guard.hpp"
#include <boost/smart_ptr.hpp>
#include <boost/thread.hpp>
#include <boost/thread/exceptions.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/bind.hpp>
namespace boost { namespace threadpool { namespace detail
{
/*! \brief Thread pool worker.
*
* A worker_thread represents a thread of execution. The worker is attached to a
* thread pool and processes tasks of that pool. The lifetime of the worker and its
* internal boost::thread is managed automatically.
*
* This class is a helper class and cannot be constructed or accessed directly.
*
* \see pool_core
*/
template <typename Pool>
class worker_thread
: public enable_shared_from_this< worker_thread<Pool> >
, private noncopyable
{
public:
typedef Pool pool_type; //!< Indicates the pool's type.
private:
shared_ptr<pool_type> m_pool; //!< Pointer to the pool which created the worker.
shared_ptr<boost::thread> m_thread; //!< Pointer to the thread which executes the run loop.
/*! Constructs a new worker.
* \param pool Pointer to it's parent pool.
* \see function create_and_attach
*/
worker_thread(shared_ptr<pool_type> const & pool)
: m_pool(pool)
{
assert(pool);
}
/*! Notifies that an exception occurred in the run loop.
*/
void died_unexpectedly()
{
m_pool->worker_died_unexpectedly(this->shared_from_this());
}
public:
/*! Executes pool's tasks sequentially.
*/
void run()
{
scope_guard notify_exception(bind(&worker_thread::died_unexpectedly, this));
while(m_pool->execute_task()) {}
notify_exception.disable();
m_pool->worker_destructed(this->shared_from_this());
}
/*! Joins the worker's thread.
*/
void join()
{
m_thread->join();
}
/*! Constructs a new worker thread and attaches it to the pool.
* \param pool Pointer to the pool.
*/
static void create_and_attach(shared_ptr<pool_type> const & pool)
{
shared_ptr<worker_thread> worker(new worker_thread(pool));
if(worker)
{
worker->m_thread.reset(new boost::thread(bind(&worker_thread::run, worker)));
}
}
};
} } } // namespace boost::threadpool::detail
#endif // THREADPOOL_DETAIL_WORKER_THREAD_HPP_INCLUDED

144
contrib/libboost-threadpool/threadpool/future.hpp
View File

@ -1,144 +0,0 @@
/*! \file
* \brief TODO.
*
* TODO.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_FUTURE_HPP_INCLUDED
#define THREADPOOL_FUTURE_HPP_INCLUDED
#include "detail/future.hpp"
#include <boost/utility/enable_if.hpp>
//#include "pool.hpp"
//#include <boost/utility.hpp>
//#include <boost/thread/mutex.hpp>
namespace boost { namespace threadpool
{
/*! \brief Experimental. Do not use in production code. TODO.
*
* TODO Future
*
* \see TODO
*
*/
template<class Result>
class future
{
private:
shared_ptr<detail::future_impl<Result> > m_impl;
public:
typedef Result const & result_type; //!< Indicates the functor's result type.
typedef Result future_result_type; //!< Indicates the future's result type.
public:
future()
: m_impl(new detail::future_impl<future_result_type>()) // TODO remove this
{
}
// only for internal usage
future(shared_ptr<detail::future_impl<Result> > const & impl)
: m_impl(impl)
{
}
bool ready() const
{
return m_impl->ready();
}
void wait() const
{
m_impl->wait();
}
bool timed_wait(boost::xtime const & timestamp) const
{
return m_impl->timed_wait(timestamp);
}
result_type operator()() // throw( thread::cancelation_exception, ... )
{
return (*m_impl)();
}
result_type get() // throw( thread::cancelation_exception, ... )
{
return (*m_impl)();
}
bool cancel()
{
return m_impl->cancel();
}
bool is_cancelled() const
{
return m_impl->is_cancelled();
}
};
template<class Pool, class Function>
typename disable_if <
is_void< typename result_of< Function() >::type >,
future< typename result_of< Function() >::type >
>::type
schedule(Pool& pool, const Function& task)
{
typedef typename result_of< Function() >::type future_result_type;
// create future impl and future
shared_ptr<detail::future_impl<future_result_type> > impl(new detail::future_impl<future_result_type>);
future <future_result_type> res(impl);
// schedule future impl
pool.schedule(detail::future_impl_task_func<detail::future_impl, Function>(task, impl));
// return future
return res;
/*
TODO
if(pool->schedule(bind(&Future::run, future)))
{
return future;
}
else
{
// construct empty future
return error_future;
}
*/
}
} } // namespace boost::threadpool
#endif // THREADPOOL_FUTURE_HPP_INCLUDED

232
contrib/libboost-threadpool/threadpool/pool.hpp
View File

@ -1,232 +0,0 @@
/*! \file
* \brief Thread pool core.
*
* This file contains the threadpool's core class: pool<Task, SchedulingPolicy>.
*
* Thread pools are a mechanism for asynchronous and parallel processing
* within the same process. The pool class provides a convenient way
* for dispatching asynchronous tasks as functions objects. The scheduling
* of these tasks can be easily controlled by using customized schedulers.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_POOL_HPP_INCLUDED
#define THREADPOOL_POOL_HPP_INCLUDED
#include <boost/ref.hpp>
#include "detail/pool_core.hpp"
#include "task_adaptors.hpp"
#include "detail/locking_ptr.hpp"
#include "scheduling_policies.hpp"
#include "size_policies.hpp"
#include "shutdown_policies.hpp"
/// The namespace threadpool contains a thread pool and related utility classes.
namespace boost { namespace threadpool
{
/*! \brief Thread pool.
*
* Thread pools are a mechanism for asynchronous and parallel processing
* within the same process. The pool class provides a convenient way
* for dispatching asynchronous tasks as functions objects. The scheduling
* of these tasks can be easily controlled by using customized schedulers.
* A task must not throw an exception.
*
* A pool is DefaultConstructible, CopyConstructible and Assignable.
* It has reference semantics; all copies of the same pool are equivalent and interchangeable.
* All operations on a pool except assignment are strongly thread safe or sequentially consistent;
* that is, the behavior of concurrent calls is as if the calls have been issued sequentially in an unspecified order.
*
* \param Task A function object which implements the operator 'void operator() (void) const'. The operator () is called by the pool to execute the task. Exceptions are ignored.
* \param SchedulingPolicy A task container which determines how tasks are scheduled. It is guaranteed that this container is accessed only by one thread at a time. The scheduler shall not throw exceptions.
*
* \remarks The pool class is thread-safe.
*
* \see Tasks: task_func, prio_task_func
* \see Scheduling policies: fifo_scheduler, lifo_scheduler, prio_scheduler
*/
template <
typename Task = task_func,
template <typename> class SchedulingPolicy = fifo_scheduler,
template <typename> class SizePolicy = static_size,
template <typename> class SizePolicyController = resize_controller,
template <typename> class ShutdownPolicy = wait_for_all_tasks
>
class thread_pool
{
typedef detail::pool_core<Task,
SchedulingPolicy,
SizePolicy,
SizePolicyController,
ShutdownPolicy> pool_core_type;
shared_ptr<pool_core_type> m_core; // pimpl idiom
shared_ptr<void> m_shutdown_controller; // If the last pool holding a pointer to the core is deleted the controller shuts the pool down.
public: // Type definitions
typedef Task task_type; //!< Indicates the task's type.
typedef SchedulingPolicy<task_type> scheduler_type; //!< Indicates the scheduler's type.
/* typedef thread_pool<Task,
SchedulingPolicy,
SizePolicy,
ShutdownPolicy > pool_type; //!< Indicates the thread pool's type.
*/
typedef SizePolicy<pool_core_type> size_policy_type;
typedef SizePolicyController<pool_core_type> size_controller_type;
public:
/*! Constructor.
* \param initial_threads The pool is immediately resized to set the specified number of threads. The pool's actual number threads depends on the SizePolicy.
*/
thread_pool(size_t initial_threads = 0)
: m_core(new pool_core_type)
, m_shutdown_controller(static_cast<void*>(0), bind(&pool_core_type::shutdown, m_core))
{
size_policy_type::init(*m_core, initial_threads);
}
/*! Gets the size controller which manages the number of threads in the pool.
* \return The size controller.
* \see SizePolicy
*/
size_controller_type size_controller()
{
return m_core->size_controller();
}
/*! Gets the number of threads in the pool.
* \return The number of threads.
*/
size_t size() const
{
return m_core->size();
}
/*! Schedules a task for asynchronous execution. The task will be executed once only.
* \param task The task function object. It should not throw execeptions.
* \return true, if the task could be scheduled and false otherwise.
*/
bool schedule(task_type const & task)
{
return m_core->schedule(task);
}
/*! Returns the number of tasks which are currently executed.
* \return The number of active tasks.
*/
size_t active() const
{
return m_core->active();
}
/*! Returns the number of tasks which are ready for execution.
* \return The number of pending tasks.
*/
size_t pending() const
{
return m_core->pending();
}
/*! Removes all pending tasks from the pool's scheduler.
*/
void clear()
{
m_core->clear();
}
/*! Indicates that there are no tasks pending.
* \return true if there are no tasks ready for execution.
* \remarks This function is more efficient that the check 'pending() == 0'.
*/
bool empty() const
{
return m_core->empty();
}
/*! The current thread of execution is blocked until the sum of all active
* and pending tasks is equal or less than a given threshold.
* \param task_threshold The maximum number of tasks in pool and scheduler.
*/
void wait(size_t task_threshold = 0) const
{
m_core->wait(task_threshold);
}
/*! The current thread of execution is blocked until the timestamp is met
* or the sum of all active and pending tasks is equal or less
* than a given threshold.
* \param timestamp The time when function returns at the latest.
* \param task_threshold The maximum number of tasks in pool and scheduler.
* \return true if the task sum is equal or less than the threshold, false otherwise.
*/
bool wait(xtime const & timestamp, size_t task_threshold = 0) const
{
return m_core->wait(timestamp, task_threshold);
}
};
/*! \brief Fifo pool.
*
* The pool's tasks are fifo scheduled task_func functors.
*
*/
typedef thread_pool<task_func, fifo_scheduler, static_size, resize_controller, wait_for_all_tasks> fifo_pool;
/*! \brief Lifo pool.
*
* The pool's tasks are lifo scheduled task_func functors.
*
*/
typedef thread_pool<task_func, lifo_scheduler, static_size, resize_controller, wait_for_all_tasks> lifo_pool;
/*! \brief Pool for prioritized task.
*
* The pool's tasks are prioritized prio_task_func functors.
*
*/
typedef thread_pool<prio_task_func, prio_scheduler, static_size, resize_controller, wait_for_all_tasks> prio_pool;
/*! \brief A standard pool.
*
* The pool's tasks are fifo scheduled task_func functors.
*
*/
typedef fifo_pool pool;
} } // namespace boost::threadpool
#endif // THREADPOOL_POOL_HPP_INCLUDED

70
contrib/libboost-threadpool/threadpool/pool_adaptors.hpp
View File

@ -1,70 +0,0 @@
/*! \file
* \brief Pool adaptors.
*
* This file contains an easy-to-use adaptor similar to a smart
* pointer for the pool class.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_POOL_ADAPTORS_HPP_INCLUDED
#define THREADPOOL_POOL_ADAPTORS_HPP_INCLUDED
#include <boost/smart_ptr.hpp>
namespace boost { namespace threadpool
{
// TODO convenience scheduling function
/*! Schedules a Runnable for asynchronous execution. A Runnable is an arbitrary class with a run()
* member function. This a convenience shorthand for pool->schedule(bind(&Runnable::run, task_object)).
* \param
* \param obj The Runnable object. The member function run() will be exectued and should not throw execeptions.
* \return true, if the task could be scheduled and false otherwise.
*/
template<typename Pool, typename Runnable>
bool schedule(Pool& pool, shared_ptr<Runnable> const & obj)
{
return pool->schedule(bind(&Runnable::run, obj));
}
/*! Schedules a task for asynchronous execution. The task will be executed once only.
* \param task The task function object.
*/
template<typename Pool>
typename enable_if <
is_void< typename result_of< typename Pool::task_type() >::type >,
bool
>::type
schedule(Pool& pool, typename Pool::task_type const & task)
{
return pool.schedule(task);
}
template<typename Pool>
typename enable_if <
is_void< typename result_of< typename Pool::task_type() >::type >,
bool
>::type
schedule(shared_ptr<Pool> const pool, typename Pool::task_type const & task)
{
return pool->schedule(task);
}
} } // namespace boost::threadpool
#endif // THREADPOOL_POOL_ADAPTORS_HPP_INCLUDED

262
contrib/libboost-threadpool/threadpool/scheduling_policies.hpp
View File

@ -1,262 +0,0 @@
/*! \file
* \brief Task scheduling policies.
*
* This file contains some fundamental scheduling policies for the pool class.
* A scheduling policy is realized by a task container which controls the access to
* the tasks. Fundamentally the container determines the order the tasks are processed
* by the thread pool.
* The task containers need not to be thread-safe because they are used by the pool
* in thread-safe way.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_SCHEDULING_POLICIES_HPP_INCLUDED
#define THREADPOOL_SCHEDULING_POLICIES_HPP_INCLUDED
#include <queue>
#include <deque>
#include "task_adaptors.hpp"
namespace boost { namespace threadpool
{
/*! \brief SchedulingPolicy which implements FIFO ordering.
*
* This container implements a FIFO scheduling policy.
* The first task to be added to the scheduler will be the first to be removed.
* The processing proceeds sequentially in the same order.
* FIFO stands for "first in, first out".
*
* \param Task A function object which implements the operator()(void).
*
*/
template <typename Task = task_func>
class fifo_scheduler
{
public:
typedef Task task_type; //!< Indicates the scheduler's task type.
protected:
std::deque<task_type> m_container; //!< Internal task container.
public:
/*! Adds a new task to the scheduler.
* \param task The task object.
* \return true, if the task could be scheduled and false otherwise.
*/
bool push(task_type const & task)
{
m_container.push_back(task);
return true;
}
/*! Removes the task which should be executed next.
*/
void pop()
{
m_container.pop_front();
}
/*! Gets the task which should be executed next.
* \return The task object to be executed.
*/
task_type const & top() const
{
return m_container.front();
}
/*! Gets the current number of tasks in the scheduler.
* \return The number of tasks.
* \remarks Prefer empty() to size() == 0 to check if the scheduler is empty.
*/
size_t size() const
{
return m_container.size();
}
/*! Checks if the scheduler is empty.
* \return true if the scheduler contains no tasks, false otherwise.
* \remarks Is more efficient than size() == 0.
*/
bool empty() const
{
return m_container.empty();
}
/*! Removes all tasks from the scheduler.
*/
void clear()
{
m_container.clear();
}
};
/*! \brief SchedulingPolicy which implements LIFO ordering.
*
* This container implements a LIFO scheduling policy.
* The last task to be added to the scheduler will be the first to be removed.
* LIFO stands for "last in, first out".
*
* \param Task A function object which implements the operator()(void).
*
*/
template <typename Task = task_func>
class lifo_scheduler
{
public:
typedef Task task_type; //!< Indicates the scheduler's task type.
protected:
std::deque<task_type> m_container; //!< Internal task container.
public:
/*! Adds a new task to the scheduler.
* \param task The task object.
* \return true, if the task could be scheduled and false otherwise.
*/
bool push(task_type const & task)
{
m_container.push_front(task);
return true;
}
/*! Removes the task which should be executed next.
*/
void pop()
{
m_container.pop_front();
}
/*! Gets the task which should be executed next.
* \return The task object to be executed.
*/
task_type const & top() const
{
return m_container.front();
}
/*! Gets the current number of tasks in the scheduler.
* \return The number of tasks.
* \remarks Prefer empty() to size() == 0 to check if the scheduler is empty.
*/
size_t size() const
{
return m_container.size();
}
/*! Checks if the scheduler is empty.
* \return true if the scheduler contains no tasks, false otherwise.
* \remarks Is more efficient than size() == 0.
*/
bool empty() const
{
return m_container.empty();
}
/*! Removes all tasks from the scheduler.
*/
void clear()
{
m_container.clear();
}
};
/*! \brief SchedulingPolicy which implements prioritized ordering.
*
* This container implements a scheduling policy based on task priorities.
* The task with highest priority will be the first to be removed.
* It must be possible to compare two tasks using operator<.
*
* \param Task A function object which implements the operator() and operator<. operator< must be a partial ordering.
*
* \see prio_thread_func
*
*/
template <typename Task = prio_task_func>
class prio_scheduler
{
public:
typedef Task task_type; //!< Indicates the scheduler's task type.
protected:
std::priority_queue<task_type> m_container; //!< Internal task container.
public:
/*! Adds a new task to the scheduler.
* \param task The task object.
* \return true, if the task could be scheduled and false otherwise.
*/
bool push(task_type const & task)
{
m_container.push(task);
return true;
}
/*! Removes the task which should be executed next.
*/
void pop()
{
m_container.pop();
}
/*! Gets the task which should be executed next.
* \return The task object to be executed.
*/
task_type const & top() const
{
return m_container.top();
}
/*! Gets the current number of tasks in the scheduler.
* \return The number of tasks.
* \remarks Prefer empty() to size() == 0 to check if the scheduler is empty.
*/
size_t size() const
{
return m_container.size();
}
/*! Checks if the scheduler is empty.
* \return true if the scheduler contains no tasks, false otherwise.
* \remarks Is more efficient than size() == 0.
*/
bool empty() const
{
return m_container.empty();
}
/*! Removes all tasks from the scheduler.
*/
void clear()
{
while(!m_container.empty())
{
m_container.pop();
}
}
};
} } // namespace boost::threadpool
#endif // THREADPOOL_SCHEDULING_POLICIES_HPP_INCLUDED

83
contrib/libboost-threadpool/threadpool/shutdown_policies.hpp
View File

@ -1,83 +0,0 @@
/*! \file
* \brief Shutdown policies.
*
* This file contains shutdown policies for thread_pool.
* A shutdown policy controls the pool's behavior from the time
* when the pool is not referenced any longer.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_SHUTDOWN_POLICIES_HPP_INCLUDED
#define THREADPOOL_SHUTDOWN_POLICIES_HPP_INCLUDED
/// The namespace threadpool contains a thread pool and related utility classes.
namespace boost { namespace threadpool
{
/*! \brief ShutdownPolicy which waits for the completion of all tasks
* and the worker termination afterwards.
*
* \param Pool The pool's core type.
*/
template<typename Pool>
class wait_for_all_tasks
{
public:
static void shutdown(Pool& pool)
{
pool.wait();
pool.terminate_all_workers(true);
}
};
/*! \brief ShutdownPolicy which waits for the completion of all active tasks
* and the worker termination afterwards.
*
* \param Pool The pool's core type.
*/
template<typename Pool>
class wait_for_active_tasks
{
public:
static void shutdown(Pool& pool)
{
pool.clear();
pool.wait();
pool.terminate_all_workers(true);
}
};
/*! \brief ShutdownPolicy which does not wait for any tasks or worker termination.
*
* This policy does not wait for any tasks. Nevertheless all active tasks will be processed completely.
*
* \param Pool The pool's core type.
*/
template<typename Pool>
class immediately
{
public:
static void shutdown(Pool& pool)
{
pool.clear();
pool.terminate_all_workers(false);
}
};
} } // namespace boost::threadpool
#endif // THREADPOOL_SHUTDOWN_POLICIES_HPP_INCLUDED

99
contrib/libboost-threadpool/threadpool/size_policies.hpp
View File

@ -1,99 +0,0 @@
/*! \file
* \brief Size policies.
*
* This file contains size policies for thread_pool. A size
* policy controls the number of worker threads in the pool.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_SIZE_POLICIES_HPP_INCLUDED
#define THREADPOOL_SIZE_POLICIES_HPP_INCLUDED
/// The namespace threadpool contains a thread pool and related utility classes.
namespace boost { namespace threadpool
{
/*! \brief SizePolicyController which provides no functionality.
*
* \param Pool The pool's core type.
*/
template<typename Pool>
struct empty_controller
{
empty_controller(typename Pool::size_policy_type&, shared_ptr<Pool>) {}
};
/*! \brief SizePolicyController which allows resizing.
*
* \param Pool The pool's core type.
*/
template< typename Pool >
class resize_controller
{
typedef typename Pool::size_policy_type size_policy_type;
reference_wrapper<size_policy_type> m_policy;
shared_ptr<Pool> m_pool; //!< to make sure that the pool is alive (the policy pointer is valid) as long as the controller exists
public:
resize_controller(size_policy_type& policy, shared_ptr<Pool> pool)
: m_policy(policy)
, m_pool(pool)
{
}
bool resize(size_t worker_count)
{
return m_policy.get().resize(worker_count);
}
};
/*! \brief SizePolicy which preserves the thread count.
*
* \param Pool The pool's core type.
*/
template<typename Pool>
class static_size
{
reference_wrapper<Pool volatile> m_pool;
public:
static void init(Pool& pool, size_t const worker_count)
{
pool.resize(worker_count);
}
static_size(Pool volatile & pool)
: m_pool(pool)
{}
bool resize(size_t const worker_count)
{
return m_pool.get().resize(worker_count);
}
void worker_died_unexpectedly(size_t const new_worker_count)
{
m_pool.get().resize(new_worker_count + 1);
}
// TODO this functions are not called yet
void task_scheduled() {}
void task_finished() {}
};
} } // namespace boost::threadpool
#endif // THREADPOOL_SIZE_POLICIES_HPP_INCLUDED

176
contrib/libboost-threadpool/threadpool/task_adaptors.hpp
View File

@ -1,176 +0,0 @@
/*! \file
* \brief Task adaptors.
*
* This file contains adaptors for task function objects.
*
* Copyright (c) 2005-2007 Philipp Henkel
*
* Use, modification, and distribution are subject to the
* Boost Software License, Version 1.0. (See accompanying file
* LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
*
* http://threadpool.sourceforge.net
*
*/
#ifndef THREADPOOL_TASK_ADAPTERS_HPP_INCLUDED
#define THREADPOOL_TASK_ADAPTERS_HPP_INCLUDED
#include <boost/version.hpp>
#if BOOST_VERSION >= 105000
#ifndef TIME_UTC
#define TIME_UTC TIME_UTC_
#endif
#endif
#include <boost/smart_ptr.hpp>
#include <boost/function.hpp>
#include <boost/thread.hpp>
namespace boost { namespace threadpool
{
/*! \brief Standard task function object.
*
* This function object wraps a nullary function which returns void.
* The wrapped function is invoked by calling the operator ().
*
* \see boost function library
*
*/
typedef function0<void> task_func;
/*! \brief Prioritized task function object.
*
* This function object wraps a task_func object and binds a priority to it.
* prio_task_funcs can be compared using the operator < which realises a partial ordering.
* The wrapped task function is invoked by calling the operator ().
*
* \see prio_scheduler
*
*/
class prio_task_func
{
private:
unsigned int m_priority; //!< The priority of the task's function.
task_func m_function; //!< The task's function.
public:
typedef void result_type; //!< Indicates the functor's result type.
public:
/*! Constructor.
* \param priority The priority of the task.
* \param function The task's function object.
*/
prio_task_func(unsigned int const priority, task_func const & function)
: m_priority(priority)
, m_function(function)
{
}
/*! Executes the task function.
*/
void operator() (void) const
{
if(m_function)
{
m_function();
}
}
/*! Comparison operator which realises a partial ordering based on priorities.
* \param rhs The object to compare with.
* \return true if the priority of *this is less than right hand side's priority, false otherwise.
*/
bool operator< (const prio_task_func& rhs) const
{
return m_priority < rhs.m_priority;
}
}; // prio_task_func
/*! \brief Looped task function object.
*
* This function object wraps a boolean thread function object.
* The wrapped task function is invoked by calling the operator () and it is executed in regular
* time intervals until false is returned. The interval length may be zero.
* Please note that a pool's thread is engaged as long as the task is looped.
*
*/
class looped_task_func
{
private:
function0<bool> m_function; //!< The task's function.
unsigned int m_break_s; //!< Duration of breaks in seconds.
unsigned int m_break_ns; //!< Duration of breaks in nano seconds.
public:
typedef void result_type; //!< Indicates the functor's result type.
public:
/*! Constructor.
* \param function The task's function object which is looped until false is returned.
* \param interval The minimum break time in milli seconds before the first execution of the task function and between the following ones.
*/
looped_task_func(function0<bool> const & function, unsigned int const interval = 0)
: m_function(function)
{
m_break_s = interval / 1000;
m_break_ns = (interval - m_break_s * 1000) * 1000 * 1000;
}
/*! Executes the task function.
*/
void operator() (void) const
{
if(m_function)
{
if(m_break_s > 0 || m_break_ns > 0)
{ // Sleep some time before first execution
xtime xt;
xtime_get(&xt, TIME_UTC);
xt.nsec += m_break_ns;
xt.sec += m_break_s;
thread::sleep(xt);
}
while(m_function())
{
if(m_break_s > 0 || m_break_ns > 0)
{
xtime xt;
xtime_get(&xt, TIME_UTC);
xt.nsec += m_break_ns;
xt.sec += m_break_s;
thread::sleep(xt);
}
else
{
thread::yield(); // Be fair to other threads
}
}
}
}
}; // looped_task_func
} } // namespace boost::threadpool
#endif // THREADPOOL_TASK_ADAPTERS_HPP_INCLUDED

2
contrib/libcpuid/include/cpuid/libcpuid_util.c
View File

@ -38,7 +38,7 @@ void match_features(const struct feature_map_t* matchtable, int count, uint32_t
{
int i;
for (i = 0; i < count; i++)
if (reg & (1 << matchtable[i].bit))
if (reg & (1U << matchtable[i].bit))
data->flags[matchtable[i].feature] = 1;
}

6
contrib/libpoco/CMakeLists.txt
View File

@ -136,7 +136,8 @@ endif(WIN32)
if (UNIX AND NOT ANDROID )
add_definitions( -DPOCO_OS_FAMILY_UNIX )
add_definitions( -Wno-unused-private-field -Wno-unused-local-typedef -Wno-for-loop-analysis -Wno-unknown-pragmas )
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field -Wno-unused-local-typedef -Wno-for-loop-analysis -Wno-unknown-pragmas -Wno-unused-variable")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unknown-pragmas -Wno-unused-variable")
# Standard 'must be' defines
if (APPLE)
add_definitions( -DPOCO_HAVE_IPv6 -DPOCO_NO_STAT64)
@ -321,9 +322,6 @@ install(
message(STATUS "CMake ${CMAKE_VERSION} successfully configured ${PROJECT_NAME} using ${CMAKE_GENERATOR} generator")
message(STATUS "Installation target path: ${CMAKE_INSTALL_PREFIX}")
message(STATUS "C_FLAGS: =${CMAKE_C_FLAGS}")
message(STATUS "CXX_FLAGS:=${CMAKE_CXX_FLAGS}")
foreach(component ${Poco_COMPONENTS})
message(STATUS "Building: ${component}")
endforeach()

4
contrib/libpoco/Foundation/include/Poco/Task.h
View File

@ -26,6 +26,8 @@
#include "Poco/Mutex.h"
#include "Poco/Event.h"
#include <atomic>
namespace Poco {
@ -140,7 +142,7 @@ private:
std::string _name;
TaskManager* _pOwner;
float _progress;
TaskState _state;
std::atomic<TaskState> _state;
Event _cancelEvent;
mutable FastMutex _mutex;

15
contrib/libpoco/Foundation/src/Task.cpp
View File

@ -17,7 +17,7 @@
#include "Poco/Task.h"
#include "Poco/TaskManager.h"
#include "Poco/Exception.h"
#include <array>
namespace Poco {
@ -61,8 +61,17 @@ void Task::run()
pOwner->taskStarted(this);
try
{
_state = TASK_RUNNING;
runTask();
/** Task can be already cancelled.
* To prevent endless executing already cancelled task _state is assigned to TASK_RUNNING only if _state != TASK_CANCELLING
*/
std::array<TaskState, 3> allowed_states{TASK_IDLE, TASK_STARTING, TASK_FINISHED};
for (auto & expected : allowed_states)
if (_state.compare_exchange_strong(expected, TASK_RUNNING))
break;
if (_state == TASK_RUNNING)
runTask();
}
catch (Exception& exc)
{

6
contrib/libpoco/Net/src/DNS.cpp
View File

@ -419,7 +419,11 @@ void DNS::aierror(int code, const std::string& arg)
throw HostNotFoundException(arg);
#endif
default:
throw DNSException("EAI", NumberFormatter::format(code));
#if defined(POCO_HAVE_ADDRINFO)
throw DNSException(gai_strerror(code), NumberFormatter::format(code), code);
#else
throw DNSException("EAI", NumberFormatter::format(code), code);
#endif
}
#endif // POCO_HAVE_IPv6 || defined(POCO_HAVE_ADDRINFO)
}

18
contrib/libzookeeper/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
add_definitions(-DHAVE_CONFIG_H -DTHREADED)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
add_definitions(-Wno-unused-but-set-variable)
endif()
include_directories (include/zookeeper src)
add_library (zookeeper
src/zookeeper.c
src/zookeeper.jute.c
src/zk_hashtable.c
src/zk_log.c
src/mt_adaptor.c
src/recordio.c
src/hashtable/hashtable.c
src/hashtable/hashtable_itr.c
)

202
contrib/libzookeeper/LICENSE Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

1
contrib/libzookeeper/README Normal file
View File

@ -0,0 +1 @@
http://apache-mirror.rbc.ru/pub/apache/zookeeper/stable/zookeeper-3.4.8.tar.gz

47
contrib/libzookeeper/include/zookeeper/proto.h Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef PROTO_H_
#define PROTO_H_
#ifdef __cplusplus
extern "C" {
#endif
#define ZOO_NOTIFY_OP 0
#define ZOO_CREATE_OP 1
#define ZOO_DELETE_OP 2
#define ZOO_EXISTS_OP 3
#define ZOO_GETDATA_OP 4
#define ZOO_SETDATA_OP 5
#define ZOO_GETACL_OP 6
#define ZOO_SETACL_OP 7
#define ZOO_GETCHILDREN_OP 8
#define ZOO_SYNC_OP 9
#define ZOO_PING_OP 11
#define ZOO_GETCHILDREN2_OP 12
#define ZOO_CHECK_OP 13
#define ZOO_MULTI_OP 14
#define ZOO_CLOSE_OP -11
#define ZOO_SETAUTH_OP 100
#define ZOO_SETWATCHES_OP 101
#ifdef __cplusplus
}
#endif
#endif /*PROTO_H_*/

82
contrib/libzookeeper/include/zookeeper/recordio.h Normal file
View File

@ -0,0 +1,82 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __RECORDIO_H__
#define __RECORDIO_H__
#include <sys/types.h>
#ifndef WIN32
#define STRUCT_INITIALIZER(l,r) .l = r
#else
#define STRUCT_INITIALIZER(l,r) r
#include "winconfig.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct buffer {
int32_t len;
char *buff;
};
void deallocate_String(char **s);
void deallocate_Buffer(struct buffer *b);
void deallocate_vector(void *d);
struct iarchive {
int (*start_record)(struct iarchive *ia, const char *tag);
int (*end_record)(struct iarchive *ia, const char *tag);
int (*start_vector)(struct iarchive *ia, const char *tag, int32_t *count);
int (*end_vector)(struct iarchive *ia, const char *tag);
int (*deserialize_Bool)(struct iarchive *ia, const char *name, int32_t *);
int (*deserialize_Int)(struct iarchive *ia, const char *name, int32_t *);
int (*deserialize_Long)(struct iarchive *ia, const char *name, int64_t *);
int (*deserialize_Buffer)(struct iarchive *ia, const char *name,
struct buffer *);
int (*deserialize_String)(struct iarchive *ia, const char *name, char **);
void *priv;
};
struct oarchive {
int (*start_record)(struct oarchive *oa, const char *tag);
int (*end_record)(struct oarchive *oa, const char *tag);
int (*start_vector)(struct oarchive *oa, const char *tag, const int32_t *count);
int (*end_vector)(struct oarchive *oa, const char *tag);
int (*serialize_Bool)(struct oarchive *oa, const char *name, const int32_t *);
int (*serialize_Int)(struct oarchive *oa, const char *name, const int32_t *);
int (*serialize_Long)(struct oarchive *oa, const char *name,
const int64_t *);
int (*serialize_Buffer)(struct oarchive *oa, const char *name,
const struct buffer *);
int (*serialize_String)(struct oarchive *oa, const char *name, char **);
void *priv;
};
struct oarchive *create_buffer_oarchive(void);
void close_buffer_oarchive(struct oarchive **oa, int free_buffer);
struct iarchive *create_buffer_iarchive(char *buffer, int len);
void close_buffer_iarchive(struct iarchive **ia);
char *get_buffer(struct oarchive *);
int get_buffer_len(struct oarchive *);
int64_t zoo_htonll(int64_t v);
#ifdef __cplusplus
}
#endif
#endif

1583
contrib/libzookeeper/include/zookeeper/zookeeper.h Normal file
View File

File diff suppressed because it is too large Load Diff

485
contrib/libzookeeper/include/zookeeper/zookeeper.jute.h Normal file
View File

@ -0,0 +1,485 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __ZOOKEEPER_JUTE__
#define __ZOOKEEPER_JUTE__
#include "recordio.h"
#ifdef __cplusplus
extern "C" {
#endif
struct Id {
char * scheme;
char * id;
};
int serialize_Id(struct oarchive *out, const char *tag, struct Id *v);
int deserialize_Id(struct iarchive *in, const char *tag, struct Id*v);
void deallocate_Id(struct Id*);
struct ACL {
int32_t perms;
struct Id id;
};
int serialize_ACL(struct oarchive *out, const char *tag, struct ACL *v);
int deserialize_ACL(struct iarchive *in, const char *tag, struct ACL*v);
void deallocate_ACL(struct ACL*);
struct Stat {
int64_t czxid;
int64_t mzxid;
int64_t ctime;
int64_t mtime;
int32_t version;
int32_t cversion;
int32_t aversion;
int64_t ephemeralOwner;
int32_t dataLength;
int32_t numChildren;
int64_t pzxid;
};
int serialize_Stat(struct oarchive *out, const char *tag, struct Stat *v);
int deserialize_Stat(struct iarchive *in, const char *tag, struct Stat*v);
void deallocate_Stat(struct Stat*);
struct StatPersisted {
int64_t czxid;
int64_t mzxid;
int64_t ctime;
int64_t mtime;
int32_t version;
int32_t cversion;
int32_t aversion;
int64_t ephemeralOwner;
int64_t pzxid;
};
int serialize_StatPersisted(struct oarchive *out, const char *tag, struct StatPersisted *v);
int deserialize_StatPersisted(struct iarchive *in, const char *tag, struct StatPersisted*v);
void deallocate_StatPersisted(struct StatPersisted*);
struct StatPersistedV1 {
int64_t czxid;
int64_t mzxid;
int64_t ctime;
int64_t mtime;
int32_t version;
int32_t cversion;
int32_t aversion;
int64_t ephemeralOwner;
};
int serialize_StatPersistedV1(struct oarchive *out, const char *tag, struct StatPersistedV1 *v);
int deserialize_StatPersistedV1(struct iarchive *in, const char *tag, struct StatPersistedV1*v);
void deallocate_StatPersistedV1(struct StatPersistedV1*);
struct ConnectRequest {
int32_t protocolVersion;
int64_t lastZxidSeen;
int32_t timeOut;
int64_t sessionId;
struct buffer passwd;
};
int serialize_ConnectRequest(struct oarchive *out, const char *tag, struct ConnectRequest *v);
int deserialize_ConnectRequest(struct iarchive *in, const char *tag, struct ConnectRequest*v);
void deallocate_ConnectRequest(struct ConnectRequest*);
struct ConnectResponse {
int32_t protocolVersion;
int32_t timeOut;
int64_t sessionId;
struct buffer passwd;
};
int serialize_ConnectResponse(struct oarchive *out, const char *tag, struct ConnectResponse *v);
int deserialize_ConnectResponse(struct iarchive *in, const char *tag, struct ConnectResponse*v);
void deallocate_ConnectResponse(struct ConnectResponse*);
struct String_vector {
int32_t count;
char * *data;
};
int serialize_String_vector(struct oarchive *out, const char *tag, struct String_vector *v);
int deserialize_String_vector(struct iarchive *in, const char *tag, struct String_vector *v);
int allocate_String_vector(struct String_vector *v, int32_t len);
int deallocate_String_vector(struct String_vector *v);
struct SetWatches {
int64_t relativeZxid;
struct String_vector dataWatches;
struct String_vector existWatches;
struct String_vector childWatches;
};
int serialize_SetWatches(struct oarchive *out, const char *tag, struct SetWatches *v);
int deserialize_SetWatches(struct iarchive *in, const char *tag, struct SetWatches*v);
void deallocate_SetWatches(struct SetWatches*);
struct RequestHeader {
int32_t xid;
int32_t type;
};
int serialize_RequestHeader(struct oarchive *out, const char *tag, struct RequestHeader *v);
int deserialize_RequestHeader(struct iarchive *in, const char *tag, struct RequestHeader*v);
void deallocate_RequestHeader(struct RequestHeader*);
struct MultiHeader {
int32_t type;
int32_t done;
int32_t err;
};
int serialize_MultiHeader(struct oarchive *out, const char *tag, struct MultiHeader *v);
int deserialize_MultiHeader(struct iarchive *in, const char *tag, struct MultiHeader*v);
void deallocate_MultiHeader(struct MultiHeader*);
struct AuthPacket {
int32_t type;
char * scheme;
struct buffer auth;
};
int serialize_AuthPacket(struct oarchive *out, const char *tag, struct AuthPacket *v);
int deserialize_AuthPacket(struct iarchive *in, const char *tag, struct AuthPacket*v);
void deallocate_AuthPacket(struct AuthPacket*);
struct ReplyHeader {
int32_t xid;
int64_t zxid;
int32_t err;
};
int serialize_ReplyHeader(struct oarchive *out, const char *tag, struct ReplyHeader *v);
int deserialize_ReplyHeader(struct iarchive *in, const char *tag, struct ReplyHeader*v);
void deallocate_ReplyHeader(struct ReplyHeader*);
struct GetDataRequest {
char * path;
int32_t watch;
};
int serialize_GetDataRequest(struct oarchive *out, const char *tag, struct GetDataRequest *v);
int deserialize_GetDataRequest(struct iarchive *in, const char *tag, struct GetDataRequest*v);
void deallocate_GetDataRequest(struct GetDataRequest*);
struct SetDataRequest {
char * path;
struct buffer data;
int32_t version;
};
int serialize_SetDataRequest(struct oarchive *out, const char *tag, struct SetDataRequest *v);
int deserialize_SetDataRequest(struct iarchive *in, const char *tag, struct SetDataRequest*v);
void deallocate_SetDataRequest(struct SetDataRequest*);
struct SetDataResponse {
struct Stat stat;
};
int serialize_SetDataResponse(struct oarchive *out, const char *tag, struct SetDataResponse *v);
int deserialize_SetDataResponse(struct iarchive *in, const char *tag, struct SetDataResponse*v);
void deallocate_SetDataResponse(struct SetDataResponse*);
struct GetSASLRequest {
struct buffer token;
};
int serialize_GetSASLRequest(struct oarchive *out, const char *tag, struct GetSASLRequest *v);
int deserialize_GetSASLRequest(struct iarchive *in, const char *tag, struct GetSASLRequest*v);
void deallocate_GetSASLRequest(struct GetSASLRequest*);
struct SetSASLRequest {
struct buffer token;
};
int serialize_SetSASLRequest(struct oarchive *out, const char *tag, struct SetSASLRequest *v);
int deserialize_SetSASLRequest(struct iarchive *in, const char *tag, struct SetSASLRequest*v);
void deallocate_SetSASLRequest(struct SetSASLRequest*);
struct SetSASLResponse {
struct buffer token;
};
int serialize_SetSASLResponse(struct oarchive *out, const char *tag, struct SetSASLResponse *v);
int deserialize_SetSASLResponse(struct iarchive *in, const char *tag, struct SetSASLResponse*v);
void deallocate_SetSASLResponse(struct SetSASLResponse*);
struct ACL_vector {
int32_t count;
struct ACL *data;
};
int serialize_ACL_vector(struct oarchive *out, const char *tag, struct ACL_vector *v);
int deserialize_ACL_vector(struct iarchive *in, const char *tag, struct ACL_vector *v);
int allocate_ACL_vector(struct ACL_vector *v, int32_t len);
int deallocate_ACL_vector(struct ACL_vector *v);
struct CreateRequest {
char * path;
struct buffer data;
struct ACL_vector acl;
int32_t flags;
};
int serialize_CreateRequest(struct oarchive *out, const char *tag, struct CreateRequest *v);
int deserialize_CreateRequest(struct iarchive *in, const char *tag, struct CreateRequest*v);
void deallocate_CreateRequest(struct CreateRequest*);
struct DeleteRequest {
char * path;
int32_t version;
};
int serialize_DeleteRequest(struct oarchive *out, const char *tag, struct DeleteRequest *v);
int deserialize_DeleteRequest(struct iarchive *in, const char *tag, struct DeleteRequest*v);
void deallocate_DeleteRequest(struct DeleteRequest*);
struct GetChildrenRequest {
char * path;
int32_t watch;
};
int serialize_GetChildrenRequest(struct oarchive *out, const char *tag, struct GetChildrenRequest *v);
int deserialize_GetChildrenRequest(struct iarchive *in, const char *tag, struct GetChildrenRequest*v);
void deallocate_GetChildrenRequest(struct GetChildrenRequest*);
struct GetChildren2Request {
char * path;
int32_t watch;
};
int serialize_GetChildren2Request(struct oarchive *out, const char *tag, struct GetChildren2Request *v);
int deserialize_GetChildren2Request(struct iarchive *in, const char *tag, struct GetChildren2Request*v);
void deallocate_GetChildren2Request(struct GetChildren2Request*);
struct CheckVersionRequest {
char * path;
int32_t version;
};
int serialize_CheckVersionRequest(struct oarchive *out, const char *tag, struct CheckVersionRequest *v);
int deserialize_CheckVersionRequest(struct iarchive *in, const char *tag, struct CheckVersionRequest*v);
void deallocate_CheckVersionRequest(struct CheckVersionRequest*);
struct GetMaxChildrenRequest {
char * path;
};
int serialize_GetMaxChildrenRequest(struct oarchive *out, const char *tag, struct GetMaxChildrenRequest *v);
int deserialize_GetMaxChildrenRequest(struct iarchive *in, const char *tag, struct GetMaxChildrenRequest*v);
void deallocate_GetMaxChildrenRequest(struct GetMaxChildrenRequest*);
struct GetMaxChildrenResponse {
int32_t max;
};
int serialize_GetMaxChildrenResponse(struct oarchive *out, const char *tag, struct GetMaxChildrenResponse *v);
int deserialize_GetMaxChildrenResponse(struct iarchive *in, const char *tag, struct GetMaxChildrenResponse*v);
void deallocate_GetMaxChildrenResponse(struct GetMaxChildrenResponse*);
struct SetMaxChildrenRequest {
char * path;
int32_t max;
};
int serialize_SetMaxChildrenRequest(struct oarchive *out, const char *tag, struct SetMaxChildrenRequest *v);
int deserialize_SetMaxChildrenRequest(struct iarchive *in, const char *tag, struct SetMaxChildrenRequest*v);
void deallocate_SetMaxChildrenRequest(struct SetMaxChildrenRequest*);
struct SyncRequest {
char * path;
};
int serialize_SyncRequest(struct oarchive *out, const char *tag, struct SyncRequest *v);
int deserialize_SyncRequest(struct iarchive *in, const char *tag, struct SyncRequest*v);
void deallocate_SyncRequest(struct SyncRequest*);
struct SyncResponse {
char * path;
};
int serialize_SyncResponse(struct oarchive *out, const char *tag, struct SyncResponse *v);
int deserialize_SyncResponse(struct iarchive *in, const char *tag, struct SyncResponse*v);
void deallocate_SyncResponse(struct SyncResponse*);
struct GetACLRequest {
char * path;
};
int serialize_GetACLRequest(struct oarchive *out, const char *tag, struct GetACLRequest *v);
int deserialize_GetACLRequest(struct iarchive *in, const char *tag, struct GetACLRequest*v);
void deallocate_GetACLRequest(struct GetACLRequest*);
struct SetACLRequest {
char * path;
struct ACL_vector acl;
int32_t version;
};
int serialize_SetACLRequest(struct oarchive *out, const char *tag, struct SetACLRequest *v);
int deserialize_SetACLRequest(struct iarchive *in, const char *tag, struct SetACLRequest*v);
void deallocate_SetACLRequest(struct SetACLRequest*);
struct SetACLResponse {
struct Stat stat;
};
int serialize_SetACLResponse(struct oarchive *out, const char *tag, struct SetACLResponse *v);
int deserialize_SetACLResponse(struct iarchive *in, const char *tag, struct SetACLResponse*v);
void deallocate_SetACLResponse(struct SetACLResponse*);
struct WatcherEvent {
int32_t type;
int32_t state;
char * path;
};
int serialize_WatcherEvent(struct oarchive *out, const char *tag, struct WatcherEvent *v);
int deserialize_WatcherEvent(struct iarchive *in, const char *tag, struct WatcherEvent*v);
void deallocate_WatcherEvent(struct WatcherEvent*);
struct ErrorResponse {
int32_t err;
};
int serialize_ErrorResponse(struct oarchive *out, const char *tag, struct ErrorResponse *v);
int deserialize_ErrorResponse(struct iarchive *in, const char *tag, struct ErrorResponse*v);
void deallocate_ErrorResponse(struct ErrorResponse*);
struct CreateResponse {
char * path;
};
int serialize_CreateResponse(struct oarchive *out, const char *tag, struct CreateResponse *v);
int deserialize_CreateResponse(struct iarchive *in, const char *tag, struct CreateResponse*v);
void deallocate_CreateResponse(struct CreateResponse*);
struct ExistsRequest {
char * path;
int32_t watch;
};
int serialize_ExistsRequest(struct oarchive *out, const char *tag, struct ExistsRequest *v);
int deserialize_ExistsRequest(struct iarchive *in, const char *tag, struct ExistsRequest*v);
void deallocate_ExistsRequest(struct ExistsRequest*);
struct ExistsResponse {
struct Stat stat;
};
int serialize_ExistsResponse(struct oarchive *out, const char *tag, struct ExistsResponse *v);
int deserialize_ExistsResponse(struct iarchive *in, const char *tag, struct ExistsResponse*v);
void deallocate_ExistsResponse(struct ExistsResponse*);
struct GetDataResponse {
struct buffer data;
struct Stat stat;
};
int serialize_GetDataResponse(struct oarchive *out, const char *tag, struct GetDataResponse *v);
int deserialize_GetDataResponse(struct iarchive *in, const char *tag, struct GetDataResponse*v);
void deallocate_GetDataResponse(struct GetDataResponse*);
struct GetChildrenResponse {
struct String_vector children;
};
int serialize_GetChildrenResponse(struct oarchive *out, const char *tag, struct GetChildrenResponse *v);
int deserialize_GetChildrenResponse(struct iarchive *in, const char *tag, struct GetChildrenResponse*v);
void deallocate_GetChildrenResponse(struct GetChildrenResponse*);
struct GetChildren2Response {
struct String_vector children;
struct Stat stat;
};
int serialize_GetChildren2Response(struct oarchive *out, const char *tag, struct GetChildren2Response *v);
int deserialize_GetChildren2Response(struct iarchive *in, const char *tag, struct GetChildren2Response*v);
void deallocate_GetChildren2Response(struct GetChildren2Response*);
struct GetACLResponse {
struct ACL_vector acl;
struct Stat stat;
};
int serialize_GetACLResponse(struct oarchive *out, const char *tag, struct GetACLResponse *v);
int deserialize_GetACLResponse(struct iarchive *in, const char *tag, struct GetACLResponse*v);
void deallocate_GetACLResponse(struct GetACLResponse*);
struct LearnerInfo {
int64_t serverid;
int32_t protocolVersion;
};
int serialize_LearnerInfo(struct oarchive *out, const char *tag, struct LearnerInfo *v);
int deserialize_LearnerInfo(struct iarchive *in, const char *tag, struct LearnerInfo*v);
void deallocate_LearnerInfo(struct LearnerInfo*);
struct Id_vector {
int32_t count;
struct Id *data;
};
int serialize_Id_vector(struct oarchive *out, const char *tag, struct Id_vector *v);
int deserialize_Id_vector(struct iarchive *in, const char *tag, struct Id_vector *v);
int allocate_Id_vector(struct Id_vector *v, int32_t len);
int deallocate_Id_vector(struct Id_vector *v);
struct QuorumPacket {
int32_t type;
int64_t zxid;
struct buffer data;
struct Id_vector authinfo;
};
int serialize_QuorumPacket(struct oarchive *out, const char *tag, struct QuorumPacket *v);
int deserialize_QuorumPacket(struct iarchive *in, const char *tag, struct QuorumPacket*v);
void deallocate_QuorumPacket(struct QuorumPacket*);
struct FileHeader {
int32_t magic;
int32_t version;
int64_t dbid;
};
int serialize_FileHeader(struct oarchive *out, const char *tag, struct FileHeader *v);
int deserialize_FileHeader(struct iarchive *in, const char *tag, struct FileHeader*v);
void deallocate_FileHeader(struct FileHeader*);
struct TxnHeader {
int64_t clientId;
int32_t cxid;
int64_t zxid;
int64_t time;
int32_t type;
};
int serialize_TxnHeader(struct oarchive *out, const char *tag, struct TxnHeader *v);
int deserialize_TxnHeader(struct iarchive *in, const char *tag, struct TxnHeader*v);
void deallocate_TxnHeader(struct TxnHeader*);
struct CreateTxnV0 {
char * path;
struct buffer data;
struct ACL_vector acl;
int32_t ephemeral;
};
int serialize_CreateTxnV0(struct oarchive *out, const char *tag, struct CreateTxnV0 *v);
int deserialize_CreateTxnV0(struct iarchive *in, const char *tag, struct CreateTxnV0*v);
void deallocate_CreateTxnV0(struct CreateTxnV0*);
struct CreateTxn {
char * path;
struct buffer data;
struct ACL_vector acl;
int32_t ephemeral;
int32_t parentCVersion;
};
int serialize_CreateTxn(struct oarchive *out, const char *tag, struct CreateTxn *v);
int deserialize_CreateTxn(struct iarchive *in, const char *tag, struct CreateTxn*v);
void deallocate_CreateTxn(struct CreateTxn*);
struct DeleteTxn {
char * path;
};
int serialize_DeleteTxn(struct oarchive *out, const char *tag, struct DeleteTxn *v);
int deserialize_DeleteTxn(struct iarchive *in, const char *tag, struct DeleteTxn*v);
void deallocate_DeleteTxn(struct DeleteTxn*);
struct SetDataTxn {
char * path;
struct buffer data;
int32_t version;
};
int serialize_SetDataTxn(struct oarchive *out, const char *tag, struct SetDataTxn *v);
int deserialize_SetDataTxn(struct iarchive *in, const char *tag, struct SetDataTxn*v);
void deallocate_SetDataTxn(struct SetDataTxn*);
struct CheckVersionTxn {
char * path;
int32_t version;
};
int serialize_CheckVersionTxn(struct oarchive *out, const char *tag, struct CheckVersionTxn *v);
int deserialize_CheckVersionTxn(struct iarchive *in, const char *tag, struct CheckVersionTxn*v);
void deallocate_CheckVersionTxn(struct CheckVersionTxn*);
struct SetACLTxn {
char * path;
struct ACL_vector acl;
int32_t version;
};
int serialize_SetACLTxn(struct oarchive *out, const char *tag, struct SetACLTxn *v);
int deserialize_SetACLTxn(struct iarchive *in, const char *tag, struct SetACLTxn*v);
void deallocate_SetACLTxn(struct SetACLTxn*);
struct SetMaxChildrenTxn {
char * path;
int32_t max;
};
int serialize_SetMaxChildrenTxn(struct oarchive *out, const char *tag, struct SetMaxChildrenTxn *v);
int deserialize_SetMaxChildrenTxn(struct iarchive *in, const char *tag, struct SetMaxChildrenTxn*v);
void deallocate_SetMaxChildrenTxn(struct SetMaxChildrenTxn*);
struct CreateSessionTxn {
int32_t timeOut;
};
int serialize_CreateSessionTxn(struct oarchive *out, const char *tag, struct CreateSessionTxn *v);
int deserialize_CreateSessionTxn(struct iarchive *in, const char *tag, struct CreateSessionTxn*v);
void deallocate_CreateSessionTxn(struct CreateSessionTxn*);
struct ErrorTxn {
int32_t err;
};
int serialize_ErrorTxn(struct oarchive *out, const char *tag, struct ErrorTxn *v);
int deserialize_ErrorTxn(struct iarchive *in, const char *tag, struct ErrorTxn*v);
void deallocate_ErrorTxn(struct ErrorTxn*);
struct Txn {
int32_t type;
struct buffer data;
};
int serialize_Txn(struct oarchive *out, const char *tag, struct Txn *v);
int deserialize_Txn(struct iarchive *in, const char *tag, struct Txn*v);
void deallocate_Txn(struct Txn*);
struct Txn_vector {
int32_t count;
struct Txn *data;
};
int serialize_Txn_vector(struct oarchive *out, const char *tag, struct Txn_vector *v);
int deserialize_Txn_vector(struct iarchive *in, const char *tag, struct Txn_vector *v);
int allocate_Txn_vector(struct Txn_vector *v, int32_t len);
int deallocate_Txn_vector(struct Txn_vector *v);
struct MultiTxn {
struct Txn_vector txns;
};
int serialize_MultiTxn(struct oarchive *out, const char *tag, struct MultiTxn *v);
int deserialize_MultiTxn(struct iarchive *in, const char *tag, struct MultiTxn*v);
void deallocate_MultiTxn(struct MultiTxn*);
#ifdef __cplusplus
}
#endif
#endif //ZOOKEEPER_JUTE__

51
contrib/libzookeeper/include/zookeeper/zookeeper_log.h Normal file
View File

@ -0,0 +1,51 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ZK_LOG_H_
#define ZK_LOG_H_
#include <zookeeper.h>
#ifdef __cplusplus
extern "C" {
#endif
extern ZOOAPI ZooLogLevel logLevel;
#define LOGSTREAM getLogStream()
#define LOG_ERROR(x) if(logLevel>=ZOO_LOG_LEVEL_ERROR) \
log_message(ZOO_LOG_LEVEL_ERROR,__LINE__,__func__,format_log_message x)
#define LOG_WARN(x) if(logLevel>=ZOO_LOG_LEVEL_WARN) \
log_message(ZOO_LOG_LEVEL_WARN,__LINE__,__func__,format_log_message x)
#define LOG_INFO(x) if(logLevel>=ZOO_LOG_LEVEL_INFO) \
log_message(ZOO_LOG_LEVEL_INFO,__LINE__,__func__,format_log_message x)
#define LOG_DEBUG(x) if(logLevel==ZOO_LOG_LEVEL_DEBUG) \
log_message(ZOO_LOG_LEVEL_DEBUG,__LINE__,__func__,format_log_message x)
ZOOAPI void log_message(ZooLogLevel curLevel, int line,const char* funcName,
const char* message);
ZOOAPI const char* format_log_message(const char* format,...);
FILE* getLogStream();
#ifdef __cplusplus
}
#endif
#endif /*ZK_LOG_H_*/

33
contrib/libzookeeper/include/zookeeper/zookeeper_version.h Normal file
View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ZOOKEEPER_VERSION_H_
#define ZOOKEEPER_VERSION_H_
#ifdef __cplusplus
extern "C" {
#endif
#define ZOO_MAJOR_VERSION 3
#define ZOO_MINOR_VERSION 4
#define ZOO_PATCH_VERSION 8
#ifdef __cplusplus
}
#endif
#endif /* ZOOKEEPER_VERSION_H_ */

149
contrib/libzookeeper/src/config.h Normal file
View File

@ -0,0 +1,149 @@
/* config.h. Generated from config.h.in by configure. */
/* config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if you have the <arpa/inet.h> header file. */
#define HAVE_ARPA_INET_H 1
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1
/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1
/* Define to 1 if you have the file `generated/zookeeper.jute.c'. */
#define HAVE_GENERATED_ZOOKEEPER_JUTE_C 1
/* Define to 1 if you have the file `generated/zookeeper.jute.h'. */
#define HAVE_GENERATED_ZOOKEEPER_JUTE_H 1
/* Define to 1 if you have the `getcwd' function. */
#define HAVE_GETCWD 1
/* Define to 1 if you have the `gethostbyname' function. */
#define HAVE_GETHOSTBYNAME 1
/* Define to 1 if you have the `gethostname' function. */
#define HAVE_GETHOSTNAME 1
/* Define to 1 if you have the `getlogin' function. */
#define HAVE_GETLOGIN 1
/* Define to 1 if you have the `getpwuid_r' function. */
#define HAVE_GETPWUID_R 1
/* Define to 1 if you have the `gettimeofday' function. */
#define HAVE_GETTIMEOFDAY 1
/* Define to 1 if you have the `getuid' function. */
#define HAVE_GETUID 1
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the `memmove' function. */
#define HAVE_MEMMOVE 1
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the `memset' function. */
#define HAVE_MEMSET 1
/* Define to 1 if you have the <netdb.h> header file. */
#define HAVE_NETDB_H 1
/* Define to 1 if you have the <netinet/in.h> header file. */
#define HAVE_NETINET_IN_H 1
/* Define to 1 if you have the `poll' function. */
#define HAVE_POLL 1
/* Define to 1 if you have the `socket' function. */
#define HAVE_SOCKET 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the `strchr' function. */
#define HAVE_STRCHR 1
/* Define to 1 if you have the `strdup' function. */
#define HAVE_STRDUP 1
/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the `strtol' function. */
#define HAVE_STRTOL 1
/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <sys/utsname.h> header file. */
#define HAVE_SYS_UTSNAME_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Define to the sub-directory where libtool stores uninstalled libraries. */
#define LT_OBJDIR ".libs/"
/* Name of package */
#define PACKAGE "zookeeper"
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "user@zookeeper.apache.org"
/* Define to the full name of this package. */
#define PACKAGE_NAME "zookeeper C client"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "zookeeper C client 3.4.8"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "zookeeper"
/* Define to the home page for this package. */
#define PACKAGE_URL ""
/* Define to the version of this package. */
#define PACKAGE_VERSION "3.4.8"
/* poll() second argument type */
#define POLL_NFDS_TYPE nfds_t
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1
/* Version number of package */
#define VERSION "3.4.8"
/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */
/* Define to `__inline__' or `__inline' if that's what the C compiler
calls it, or to nothing if 'inline' is not supported under any name. */
#ifndef __cplusplus
/* #undef inline */
#endif

30
contrib/libzookeeper/src/hashtable/LICENSE.txt Normal file
View File

@ -0,0 +1,30 @@
Copyright (c) 2002, 2004, Christopher Clark
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the original author; nor the names of any contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

274
contrib/libzookeeper/src/hashtable/hashtable.c Normal file
View File

@ -0,0 +1,274 @@
/* Copyright (C) 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
#include "hashtable.h"
#include "hashtable_private.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
/*
Credit for primes table: Aaron Krowne
http://br.endernet.org/~akrowne/
http://planetmath.org/encyclopedia/GoodHashTablePrimes.html
*/
static const unsigned int primes[] = {
53, 97, 193, 389,
769, 1543, 3079, 6151,
12289, 24593, 49157, 98317,
196613, 393241, 786433, 1572869,
3145739, 6291469, 12582917, 25165843,
50331653, 100663319, 201326611, 402653189,
805306457, 1610612741
};
const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]);
const float max_load_factor = 0.65;
/*****************************************************************************/
struct hashtable *
create_hashtable(unsigned int minsize,
unsigned int (*hashf) (void*),
int (*eqf) (void*,void*))
{
struct hashtable *h;
unsigned int pindex, size = primes[0];
/* Check requested hashtable isn't too large */
if (minsize > (1u << 30)) return NULL;
/* Enforce size as prime */
for (pindex=0; pindex < prime_table_length; pindex++) {
if (primes[pindex] > minsize) { size = primes[pindex]; break; }
}
h = (struct hashtable *)malloc(sizeof(struct hashtable));
if (NULL == h) return NULL; /*oom*/
h->table = (struct entry **)malloc(sizeof(struct entry*) * size);
if (NULL == h->table) { free(h); return NULL; } /*oom*/
memset(h->table, 0, size * sizeof(struct entry *));
h->tablelength = size;
h->primeindex = pindex;
h->entrycount = 0;
h->hashfn = hashf;
h->eqfn = eqf;
h->loadlimit = (unsigned int) ceil(size * max_load_factor);
return h;
}
/*****************************************************************************/
unsigned int
hash(struct hashtable *h, void *k)
{
/* Aim to protect against poor hash functions by adding logic here
* - logic taken from java 1.4 hashtable source */
unsigned int i = h->hashfn(k);
i += ~(i << 9);
i ^= ((i >> 14) | (i << 18)); /* >>> */
i += (i << 4);
i ^= ((i >> 10) | (i << 22)); /* >>> */
return i;
}
/*****************************************************************************/
static int
hashtable_expand(struct hashtable *h)
{
/* Double the size of the table to accomodate more entries */
struct entry **newtable;
struct entry *e;
struct entry **pE;
unsigned int newsize, i, index;
/* Check we're not hitting max capacity */
if (h->primeindex == (prime_table_length - 1)) return 0;
newsize = primes[++(h->primeindex)];
newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize);
if (NULL != newtable)
{
memset(newtable, 0, newsize * sizeof(struct entry *));
/* This algorithm is not 'stable'. ie. it reverses the list
* when it transfers entries between the tables */
for (i = 0; i < h->tablelength; i++) {
while (NULL != (e = h->table[i])) {
h->table[i] = e->next;
index = indexFor(newsize,e->h);
e->next = newtable[index];
newtable[index] = e;
}
}
free(h->table);
h->table = newtable;
}
/* Plan B: realloc instead */
else
{
newtable = (struct entry **)
realloc(h->table, newsize * sizeof(struct entry *));
if (NULL == newtable) { (h->primeindex)--; return 0; }
h->table = newtable;
memset(newtable[h->tablelength], 0, newsize - h->tablelength);
for (i = 0; i < h->tablelength; i++) {
for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) {
index = indexFor(newsize,e->h);
if (index == i)
{
pE = &(e->next);
}
else
{
*pE = e->next;
e->next = newtable[index];
newtable[index] = e;
}
}
}
}
h->tablelength = newsize;
h->loadlimit = (unsigned int) ceil(newsize * max_load_factor);
return -1;
}
/*****************************************************************************/
unsigned int
hashtable_count(struct hashtable *h)
{
return h->entrycount;
}
/*****************************************************************************/
int
hashtable_insert(struct hashtable *h, void *k, void *v)
{
/* This method allows duplicate keys - but they shouldn't be used */
unsigned int index;
struct entry *e;
if (++(h->entrycount) > h->loadlimit)
{
/* Ignore the return value. If expand fails, we should
* still try cramming just this value into the existing table
* -- we may not have memory for a larger table, but one more
* element may be ok. Next time we insert, we'll try expanding again.*/
hashtable_expand(h);
}
e = (struct entry *)malloc(sizeof(struct entry));
if (NULL == e) { --(h->entrycount); return 0; } /*oom*/
e->h = hash(h,k);
index = indexFor(h->tablelength,e->h);
e->k = k;
e->v = v;
e->next = h->table[index];
h->table[index] = e;
return -1;
}
/*****************************************************************************/
void * /* returns value associated with key */
hashtable_search(struct hashtable *h, void *k)
{
struct entry *e;
unsigned int hashvalue, index;
hashvalue = hash(h,k);
index = indexFor(h->tablelength,hashvalue);
e = h->table[index];
while (NULL != e)
{
/* Check hash value to short circuit heavier comparison */
if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v;
e = e->next;
}
return NULL;
}
/*****************************************************************************/
void * /* returns value associated with key */
hashtable_remove(struct hashtable *h, void *k)
{
/* TODO: consider compacting the table when the load factor drops enough,
* or provide a 'compact' method. */
struct entry *e;
struct entry **pE;
void *v;
unsigned int hashvalue, index;
hashvalue = hash(h,k);
index = indexFor(h->tablelength,hash(h,k));
pE = &(h->table[index]);
e = *pE;
while (NULL != e)
{
/* Check hash value to short circuit heavier comparison */
if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
{
*pE = e->next;
h->entrycount--;
v = e->v;
freekey(e->k);
free(e);
return v;
}
pE = &(e->next);
e = e->next;
}
return NULL;
}
/*****************************************************************************/
/* destroy */
void
hashtable_destroy(struct hashtable *h, int free_values)
{
unsigned int i;
struct entry *e, *f;
struct entry **table = h->table;
if (free_values)
{
for (i = 0; i < h->tablelength; i++)
{
e = table[i];
while (NULL != e)
{ f = e; e = e->next; freekey(f->k); free(f->v); free(f); }
}
}
else
{
for (i = 0; i < h->tablelength; i++)
{
e = table[i];
while (NULL != e)
{ f = e; e = e->next; freekey(f->k); free(f); }
}
}
free(h->table);
free(h);
}
/*
* Copyright (c) 2002, Christopher Clark
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of the original author; nor the names of any contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

209
contrib/libzookeeper/src/hashtable/hashtable.h Normal file
View File

@ -0,0 +1,209 @@
/* Copyright (C) 2002 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
#ifndef __HASHTABLE_CWC22_H__
#define __HASHTABLE_CWC22_H__
#ifdef WIN32
#include "winconfig.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct hashtable;
/* Example of use:
*
* struct hashtable *h;
* struct some_key *k;
* struct some_value *v;
*
* static unsigned int hash_from_key_fn( void *k );
* static int keys_equal_fn ( void *key1, void *key2 );
*
* h = create_hashtable(16, hash_from_key_fn, keys_equal_fn);
* k = (struct some_key *) malloc(sizeof(struct some_key));
* v = (struct some_value *) malloc(sizeof(struct some_value));
*
* (initialise k and v to suitable values)
*
* if (! hashtable_insert(h,k,v) )
* { exit(-1); }
*
* if (NULL == (found = hashtable_search(h,k) ))
* { printf("not found!"); }
*
* if (NULL == (found = hashtable_remove(h,k) ))
* { printf("Not found\n"); }
*
*/
/* Macros may be used to define type-safe(r) hashtable access functions, with
* methods specialized to take known key and value types as parameters.
*
* Example:
*
* Insert this at the start of your file:
*
* DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value);
* DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value);
* DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value);
*
* This defines the functions 'insert_some', 'search_some' and 'remove_some'.
* These operate just like hashtable_insert etc., with the same parameters,
* but their function signatures have 'struct some_key *' rather than
* 'void *', and hence can generate compile time errors if your program is
* supplying incorrect data as a key (and similarly for value).
*
* Note that the hash and key equality functions passed to create_hashtable
* still take 'void *' parameters instead of 'some key *'. This shouldn't be
* a difficult issue as they're only defined and passed once, and the other
* functions will ensure that only valid keys are supplied to them.
*
* The cost for this checking is increased code size and runtime overhead
* - if performance is important, it may be worth switching back to the
* unsafe methods once your program has been debugged with the safe methods.
* This just requires switching to some simple alternative defines - eg:
* #define insert_some hashtable_insert
*
*/
/*****************************************************************************
* create_hashtable
* @name create_hashtable
* @param minsize minimum initial size of hashtable
* @param hashfunction function for hashing keys
* @param key_eq_fn function for determining key equality
* @return newly created hashtable or NULL on failure
*/
struct hashtable *
create_hashtable(unsigned int minsize,
unsigned int (*hashfunction) (void*),
int (*key_eq_fn) (void*,void*));
/*****************************************************************************
* hashtable_insert
* @name hashtable_insert
* @param h the hashtable to insert into
* @param k the key - hashtable claims ownership and will free on removal
* @param v the value - does not claim ownership
* @return non-zero for successful insertion
*
* This function will cause the table to expand if the insertion would take
* the ratio of entries to table size over the maximum load factor.
*
* This function does not check for repeated insertions with a duplicate key.
* The value returned when using a duplicate key is undefined -- when
* the hashtable changes size, the order of retrieval of duplicate key
* entries is reversed.
* If in doubt, remove before insert.
*/
int
hashtable_insert(struct hashtable *h, void *k, void *v);
#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \
int fnname (struct hashtable *h, keytype *k, valuetype *v) \
{ \
return hashtable_insert(h,k,v); \
}
/*****************************************************************************
* hashtable_search
* @name hashtable_search
* @param h the hashtable to search
* @param k the key to search for - does not claim ownership
* @return the value associated with the key, or NULL if none found
*/
void *
hashtable_search(struct hashtable *h, void *k);
#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \
valuetype * fnname (struct hashtable *h, keytype *k) \
{ \
return (valuetype *) (hashtable_search(h,k)); \
}
/*****************************************************************************
* hashtable_remove
* @name hashtable_remove
* @param h the hashtable to remove the item from
* @param k the key to search for - does not claim ownership
* @return the value associated with the key, or NULL if none found
*/
void * /* returns value */
hashtable_remove(struct hashtable *h, void *k);
#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \
valuetype * fnname (struct hashtable *h, keytype *k) \
{ \
return (valuetype *) (hashtable_remove(h,k)); \
}
/*****************************************************************************
* hashtable_count
* @name hashtable_count
* @param h the hashtable
* @return the number of items stored in the hashtable
*/
unsigned int
hashtable_count(struct hashtable *h);
/*****************************************************************************
* hashtable_destroy
* @name hashtable_destroy
* @param h the hashtable
* @param free_values whether to call 'free' on the remaining values
*/
void
hashtable_destroy(struct hashtable *h, int free_values);
#ifdef __cplusplus
}
#endif
#endif /* __HASHTABLE_CWC22_H__ */
/*
* Copyright (c) 2002, Christopher Clark
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of the original author; nor the names of any contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

176
contrib/libzookeeper/src/hashtable/hashtable_itr.c Normal file
View File

@ -0,0 +1,176 @@
/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
#include "hashtable.h"
#include "hashtable_private.h"
#include "hashtable_itr.h"
#include <stdlib.h> /* defines NULL */
/*****************************************************************************/
/* hashtable_iterator - iterator constructor */
struct hashtable_itr *
hashtable_iterator(struct hashtable *h)
{
unsigned int i, tablelength;
struct hashtable_itr *itr = (struct hashtable_itr *)
malloc(sizeof(struct hashtable_itr));
if (NULL == itr) return NULL;
itr->h = h;
itr->e = NULL;
itr->parent = NULL;
tablelength = h->tablelength;
itr->index = tablelength;
if (0 == h->entrycount) return itr;
for (i = 0; i < tablelength; i++)
{
if (NULL != h->table[i])
{
itr->e = h->table[i];
itr->index = i;
break;
}
}
return itr;
}
/*****************************************************************************/
/* advance - advance the iterator to the next element
* returns zero if advanced to end of table */
int
hashtable_iterator_advance(struct hashtable_itr *itr)
{
unsigned int j,tablelength;
struct entry **table;
struct entry *next;
if (NULL == itr->e) return 0; /* stupidity check */
next = itr->e->next;
if (NULL != next)
{
itr->parent = itr->e;
itr->e = next;
return -1;
}
tablelength = itr->h->tablelength;
itr->parent = NULL;
if (tablelength <= (j = ++(itr->index)))
{
itr->e = NULL;
return 0;
}
table = itr->h->table;
while (NULL == (next = table[j]))
{
if (++j >= tablelength)
{
itr->index = tablelength;
itr->e = NULL;
return 0;
}
}
itr->index = j;
itr->e = next;
return -1;
}
/*****************************************************************************/
/* remove - remove the entry at the current iterator position
* and advance the iterator, if there is a successive
* element.
* If you want the value, read it before you remove:
* beware memory leaks if you don't.
* Returns zero if end of iteration. */
int
hashtable_iterator_remove(struct hashtable_itr *itr)
{
struct entry *remember_e, *remember_parent;
int ret;
/* Do the removal */
if (NULL == (itr->parent))
{
/* element is head of a chain */
itr->h->table[itr->index] = itr->e->next;
} else {
/* element is mid-chain */
itr->parent->next = itr->e->next;
}
/* itr->e is now outside the hashtable */
remember_e = itr->e;
itr->h->entrycount--;
freekey(remember_e->k);
/* Advance the iterator, correcting the parent */
remember_parent = itr->parent;
ret = hashtable_iterator_advance(itr);
if (itr->parent == remember_e) { itr->parent = remember_parent; }
free(remember_e);
return ret;
}
/*****************************************************************************/
int /* returns zero if not found */
hashtable_iterator_search(struct hashtable_itr *itr,
struct hashtable *h, void *k)
{
struct entry *e, *parent;
unsigned int hashvalue, index;
hashvalue = hash(h,k);
index = indexFor(h->tablelength,hashvalue);
e = h->table[index];
parent = NULL;
while (NULL != e)
{
/* Check hash value to short circuit heavier comparison */
if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
{
itr->index = index;
itr->e = e;
itr->parent = parent;
itr->h = h;
return -1;
}
parent = e;
e = e->next;
}
return 0;
}
/*
* Copyright (c) 2002, 2004, Christopher Clark
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of the original author; nor the names of any contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

119
contrib/libzookeeper/src/hashtable/hashtable_itr.h Normal file
View File

@ -0,0 +1,119 @@
/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
#ifndef __HASHTABLE_ITR_CWC22__
#define __HASHTABLE_ITR_CWC22__
#include "hashtable.h"
#include "hashtable_private.h" /* needed to enable inlining */
#ifdef __cplusplus
extern "C" {
#endif
/*****************************************************************************/
/* This struct is only concrete here to allow the inlining of two of the
* accessor functions. */
struct hashtable_itr
{
struct hashtable *h;
struct entry *e;
struct entry *parent;
unsigned int index;
};
/*****************************************************************************/
/* hashtable_iterator
*/
struct hashtable_itr *
hashtable_iterator(struct hashtable *h);
/*****************************************************************************/
/* hashtable_iterator_key
* - return the value of the (key,value) pair at the current position */
static inline void *
hashtable_iterator_key(struct hashtable_itr *i)
{
return i->e->k;
}
/*****************************************************************************/
/* value - return the value of the (key,value) pair at the current position */
static inline void *
hashtable_iterator_value(struct hashtable_itr *i)
{
return i->e->v;
}
/*****************************************************************************/
/* advance - advance the iterator to the next element
* returns zero if advanced to end of table */
int
hashtable_iterator_advance(struct hashtable_itr *itr);
/*****************************************************************************/
/* remove - remove current element and advance the iterator to the next element
* NB: if you need the value to free it, read it before
* removing. ie: beware memory leaks!
* returns zero if advanced to end of table */
int
hashtable_iterator_remove(struct hashtable_itr *itr);
/*****************************************************************************/
/* search - overwrite the supplied iterator, to point to the entry
* matching the supplied key.
h points to the hashtable to be searched.
* returns zero if not found. */
int
hashtable_iterator_search(struct hashtable_itr *itr,
struct hashtable *h, void *k);
#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \
int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \
{ \
return (hashtable_iterator_search(i,h,k)); \
}
#ifdef __cplusplus
}
#endif
#endif /* __HASHTABLE_ITR_CWC22__*/
/*
* Copyright (c) 2002, 2004, Christopher Clark
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of the original author; nor the names of any contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

85
contrib/libzookeeper/src/hashtable/hashtable_private.h Normal file
View File

@ -0,0 +1,85 @@
/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */
#ifndef __HASHTABLE_PRIVATE_CWC22_H__
#define __HASHTABLE_PRIVATE_CWC22_H__
#include "hashtable.h"
/*****************************************************************************/
struct entry
{
void *k, *v;
unsigned int h;
struct entry *next;
};
struct hashtable {
unsigned int tablelength;
struct entry **table;
unsigned int entrycount;
unsigned int loadlimit;
unsigned int primeindex;
unsigned int (*hashfn) (void *k);
int (*eqfn) (void *k1, void *k2);
};
/*****************************************************************************/
unsigned int
hash(struct hashtable *h, void *k);
/*****************************************************************************/
/* indexFor */
static inline unsigned int
indexFor(unsigned int tablelength, unsigned int hashvalue) {
return (hashvalue % tablelength);
};
/* Only works if tablelength == 2^N */
/*static inline unsigned int
indexFor(unsigned int tablelength, unsigned int hashvalue)
{
return (hashvalue & (tablelength - 1u));
}
*/
/*****************************************************************************/
#define freekey(X) free(X)
/*define freekey(X) ; */
/*****************************************************************************/
#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/
/*
* Copyright (c) 2002, Christopher Clark
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of the original author; nor the names of any contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

536
contrib/libzookeeper/src/mt_adaptor.c Normal file
View File

@ -0,0 +1,536 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef THREADED
#define THREADED
#endif
#ifndef DLL_EXPORT
# define USE_STATIC_LIB
#endif
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include "zk_adaptor.h"
#include "zookeeper_log.h"
#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <fcntl.h>
#include <assert.h>
#include <errno.h>
#ifndef WIN32
#include <signal.h>
#include <poll.h>
#include <unistd.h>
#include <sys/time.h>
#endif
int zoo_lock_auth(zhandle_t *zh)
{
return pthread_mutex_lock(&zh->auth_h.lock);
}
int zoo_unlock_auth(zhandle_t *zh)
{
return pthread_mutex_unlock(&zh->auth_h.lock);
}
int lock_buffer_list(buffer_head_t *l)
{
return pthread_mutex_lock(&l->lock);
}
int unlock_buffer_list(buffer_head_t *l)
{
return pthread_mutex_unlock(&l->lock);
}
int lock_completion_list(completion_head_t *l)
{
return pthread_mutex_lock(&l->lock);
}
int unlock_completion_list(completion_head_t *l)
{
pthread_cond_broadcast(&l->cond);
return pthread_mutex_unlock(&l->lock);
}
struct sync_completion *alloc_sync_completion(void)
{
struct sync_completion *sc = (struct sync_completion*)calloc(1, sizeof(struct sync_completion));
if (sc) {
pthread_cond_init(&sc->cond, 0);
pthread_mutex_init(&sc->lock, 0);
}
return sc;
}
int wait_sync_completion(struct sync_completion *sc)
{
pthread_mutex_lock(&sc->lock);
while (!sc->complete) {
pthread_cond_wait(&sc->cond, &sc->lock);
}
pthread_mutex_unlock(&sc->lock);
return 0;
}
void free_sync_completion(struct sync_completion *sc)
{
if (sc) {
pthread_mutex_destroy(&sc->lock);
pthread_cond_destroy(&sc->cond);
free(sc);
}
}
void notify_sync_completion(struct sync_completion *sc)
{
pthread_mutex_lock(&sc->lock);
sc->complete = 1;
pthread_cond_broadcast(&sc->cond);
pthread_mutex_unlock(&sc->lock);
}
int process_async(int outstanding_sync)
{
return 0;
}
#ifdef WIN32
unsigned __stdcall do_io( void * );
unsigned __stdcall do_completion( void * );
int handle_error(SOCKET sock, char* message)
{
LOG_ERROR(("%s. %d",message, WSAGetLastError()));
closesocket (sock);
return -1;
}
//--create socket pair for interupting selects.
int create_socket_pair(SOCKET fds[2])
{
struct sockaddr_in inaddr;
struct sockaddr addr;
int yes=1;
int len=0;
SOCKET lst=socket(AF_INET, SOCK_STREAM,IPPROTO_TCP);
if (lst == INVALID_SOCKET ){
LOG_ERROR(("Error creating socket. %d",WSAGetLastError()));
return -1;
}
memset(&inaddr, 0, sizeof(inaddr));
memset(&addr, 0, sizeof(addr));
inaddr.sin_family = AF_INET;
inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
inaddr.sin_port = 0; //--system assigns the port
if ( setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)) == SOCKET_ERROR ) {
return handle_error(lst,"Error trying to set socket option.");
}
if (bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)) == SOCKET_ERROR){
return handle_error(lst,"Error trying to bind socket.");
}
if (listen(lst,1) == SOCKET_ERROR){
return handle_error(lst,"Error trying to listen on socket.");
}
len=sizeof(inaddr);
getsockname(lst, &addr,&len);
fds[0]=socket(AF_INET, SOCK_STREAM,0);
if (connect(fds[0],&addr,len) == SOCKET_ERROR){
return handle_error(lst, "Error while connecting to socket.");
}
if ((fds[1]=accept(lst,0,0)) == INVALID_SOCKET){
closesocket(fds[0]);
return handle_error(lst, "Error while accepting socket connection.");
}
closesocket(lst);
return 0;
}
#else
void *do_io(void *);
void *do_completion(void *);
#endif
int wakeup_io_thread(zhandle_t *zh);
#ifdef WIN32
static int set_nonblock(SOCKET fd){
ULONG nonblocking_flag = 1;
if (ioctlsocket(fd, FIONBIO, &nonblocking_flag) == 0)
return 1;
else
return -1;
}
#else
static int set_nonblock(int fd){
long l = fcntl(fd, F_GETFL);
if(l & O_NONBLOCK) return 0;
return fcntl(fd, F_SETFL, l | O_NONBLOCK);
}
#endif
void wait_for_others(zhandle_t* zh)
{
struct adaptor_threads* adaptor=zh->adaptor_priv;
pthread_mutex_lock(&adaptor->lock);
while(adaptor->threadsToWait>0)
pthread_cond_wait(&adaptor->cond,&adaptor->lock);
pthread_mutex_unlock(&adaptor->lock);
}
void notify_thread_ready(zhandle_t* zh)
{
struct adaptor_threads* adaptor=zh->adaptor_priv;
pthread_mutex_lock(&adaptor->lock);
adaptor->threadsToWait--;
pthread_cond_broadcast(&adaptor->cond);
while(adaptor->threadsToWait>0)
pthread_cond_wait(&adaptor->cond,&adaptor->lock);
pthread_mutex_unlock(&adaptor->lock);
}
void start_threads(zhandle_t* zh)
{
int rc = 0;
struct adaptor_threads* adaptor=zh->adaptor_priv;
pthread_cond_init(&adaptor->cond,0);
pthread_mutex_init(&adaptor->lock,0);
adaptor->threadsToWait=2; // wait for 2 threads before opening the barrier
// use api_prolog() to make sure zhandle doesn't get destroyed
// while initialization is in progress
api_prolog(zh);
LOG_DEBUG(("starting threads..."));
rc=pthread_create(&adaptor->io, 0, do_io, zh);
assert("pthread_create() failed for the IO thread"&&!rc);
rc=pthread_create(&adaptor->completion, 0, do_completion, zh);
assert("pthread_create() failed for the completion thread"&&!rc);
wait_for_others(zh);
api_epilog(zh, 0);
}
int adaptor_init(zhandle_t *zh)
{
pthread_mutexattr_t recursive_mx_attr;
struct adaptor_threads *adaptor_threads = calloc(1, sizeof(*adaptor_threads));
if (!adaptor_threads) {
LOG_ERROR(("Out of memory"));
return -1;
}
/* We use a pipe for interrupting select() in unix/sol and socketpair in windows. */
#ifdef WIN32
if (create_socket_pair(adaptor_threads->self_pipe) == -1){
LOG_ERROR(("Can't make a socket."));
#else
if(pipe(adaptor_threads->self_pipe)==-1) {
LOG_ERROR(("Can't make a pipe %d",errno));
#endif
free(adaptor_threads);
return -1;
}
set_nonblock(adaptor_threads->self_pipe[1]);
set_nonblock(adaptor_threads->self_pipe[0]);
pthread_mutex_init(&zh->auth_h.lock,0);
zh->adaptor_priv = adaptor_threads;
pthread_mutex_init(&zh->to_process.lock,0);
pthread_mutex_init(&adaptor_threads->zh_lock,0);
// to_send must be recursive mutex
pthread_mutexattr_init(&recursive_mx_attr);
pthread_mutexattr_settype(&recursive_mx_attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&zh->to_send.lock,&recursive_mx_attr);
pthread_mutexattr_destroy(&recursive_mx_attr);
pthread_mutex_init(&zh->sent_requests.lock,0);
pthread_cond_init(&zh->sent_requests.cond,0);
pthread_mutex_init(&zh->completions_to_process.lock,0);
pthread_cond_init(&zh->completions_to_process.cond,0);
start_threads(zh);
return 0;
}
void adaptor_finish(zhandle_t *zh)
{
struct adaptor_threads *adaptor_threads;
// make sure zh doesn't get destroyed until after we're done here
api_prolog(zh);
adaptor_threads = zh->adaptor_priv;
if(adaptor_threads==0) {
api_epilog(zh,0);
return;
}
if(!pthread_equal(adaptor_threads->io,pthread_self())){
wakeup_io_thread(zh);
pthread_join(adaptor_threads->io, 0);
}else
pthread_detach(adaptor_threads->io);
if(!pthread_equal(adaptor_threads->completion,pthread_self())){
pthread_mutex_lock(&zh->completions_to_process.lock);
pthread_cond_broadcast(&zh->completions_to_process.cond);
pthread_mutex_unlock(&zh->completions_to_process.lock);
pthread_join(adaptor_threads->completion, 0);
}else
pthread_detach(adaptor_threads->completion);
api_epilog(zh,0);
}
void adaptor_destroy(zhandle_t *zh)
{
struct adaptor_threads *adaptor = zh->adaptor_priv;
if(adaptor==0) return;
pthread_cond_destroy(&adaptor->cond);
pthread_mutex_destroy(&adaptor->lock);
pthread_mutex_destroy(&zh->to_process.lock);
pthread_mutex_destroy(&zh->to_send.lock);
pthread_mutex_destroy(&zh->sent_requests.lock);
pthread_cond_destroy(&zh->sent_requests.cond);
pthread_mutex_destroy(&zh->completions_to_process.lock);
pthread_cond_destroy(&zh->completions_to_process.cond);
pthread_mutex_destroy(&adaptor->zh_lock);
pthread_mutex_destroy(&zh->auth_h.lock);
close(adaptor->self_pipe[0]);
close(adaptor->self_pipe[1]);
free(adaptor);
zh->adaptor_priv=0;
}
int wakeup_io_thread(zhandle_t *zh)
{
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
char c=0;
#ifndef WIN32
return write(adaptor_threads->self_pipe[1],&c,1)==1? ZOK: ZSYSTEMERROR;
#else
return send(adaptor_threads->self_pipe[1], &c, 1, 0)==1? ZOK: ZSYSTEMERROR;
#endif
}
int adaptor_send_queue(zhandle_t *zh, int timeout)
{
if(!zh->close_requested)
return wakeup_io_thread(zh);
// don't rely on the IO thread to send the messages if the app has
// requested to close
return flush_send_queue(zh, timeout);
}
/* These two are declared here because we will run the event loop
* and not the client */
#ifdef WIN32
int zookeeper_interest(zhandle_t *zh, SOCKET *fd, int *interest,
struct timeval *tv);
#else
int zookeeper_interest(zhandle_t *zh, int *fd, int *interest,
struct timeval *tv);
#endif
int zookeeper_process(zhandle_t *zh, int events);
#ifdef WIN32
unsigned __stdcall do_io( void * v)
#else
void *do_io(void *v)
#endif
{
zhandle_t *zh = (zhandle_t*)v;
#ifndef WIN32
struct pollfd fds[2];
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started IO thread"));
fds[0].fd=adaptor_threads->self_pipe[0];
fds[0].events=POLLIN;
while(!zh->close_requested) {
struct timeval tv;
int fd;
int interest;
int timeout;
int maxfd=1;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
fds[1].fd=fd;
fds[1].events=(interest&ZOOKEEPER_READ)?POLLIN:0;
fds[1].events|=(interest&ZOOKEEPER_WRITE)?POLLOUT:0;
maxfd=2;
}
timeout=tv.tv_sec * 1000 + (tv.tv_usec/1000);
poll(fds,maxfd,timeout);
if (fd != -1) {
interest=(fds[1].revents&POLLIN)?ZOOKEEPER_READ:0;
interest|=((fds[1].revents&POLLOUT)||(fds[1].revents&POLLHUP))?ZOOKEEPER_WRITE:0;
}
if(fds[0].revents&POLLIN){
// flush the pipe
char b[128];
while(read(adaptor_threads->self_pipe[0],b,sizeof(b))==sizeof(b)){}
}
#else
fd_set rfds, wfds, efds;
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started IO thread"));
FD_ZERO(&rfds); FD_ZERO(&wfds); FD_ZERO(&efds);
while(!zh->close_requested) {
struct timeval tv;
SOCKET fd;
SOCKET maxfd=adaptor_threads->self_pipe[0];
int interest;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest&ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest&ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
}
FD_SET( adaptor_threads->self_pipe[0] ,&rfds );
rc = select((int)maxfd, &rfds, &wfds, &efds, &tv);
if (fd != -1)
{
interest = (FD_ISSET(fd, &rfds))? ZOOKEEPER_READ:0;
interest|= (FD_ISSET(fd, &wfds))? ZOOKEEPER_WRITE:0;
}
if (FD_ISSET(adaptor_threads->self_pipe[0], &rfds)){
// flush the pipe/socket
char b[128];
while(recv(adaptor_threads->self_pipe[0],b,sizeof(b), 0)==sizeof(b)){}
}
#endif
// dispatch zookeeper events
rc = zookeeper_process(zh, interest);
// check the current state of the zhandle and terminate
// if it is_unrecoverable()
if(is_unrecoverable(zh))
break;
}
api_epilog(zh, 0);
LOG_DEBUG(("IO thread terminated"));
return 0;
}
#ifdef WIN32
unsigned __stdcall do_completion( void * v)
#else
void *do_completion(void *v)
#endif
{
zhandle_t *zh = v;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started completion thread"));
while(!zh->close_requested) {
pthread_mutex_lock(&zh->completions_to_process.lock);
while(!zh->completions_to_process.head && !zh->close_requested) {
pthread_cond_wait(&zh->completions_to_process.cond, &zh->completions_to_process.lock);
}
pthread_mutex_unlock(&zh->completions_to_process.lock);
process_completions(zh);
}
api_epilog(zh, 0);
LOG_DEBUG(("completion thread terminated"));
return 0;
}
int32_t inc_ref_counter(zhandle_t* zh,int i)
{
int incr=(i<0?-1:(i>0?1:0));
// fetch_and_add implements atomic post-increment
int v=fetch_and_add(&zh->ref_counter,incr);
// inc_ref_counter wants pre-increment
v+=incr; // simulate pre-increment
return v;
}
int32_t fetch_and_add(volatile int32_t* operand, int incr)
{
#ifndef WIN32
int32_t result;
asm __volatile__(
"lock xaddl %0,%1\n"
: "=r"(result), "=m"(*(int *)operand)
: "0"(incr)
: "memory");
return result;
#else
volatile int32_t result;
_asm
{
mov eax, operand; //eax = v;
mov ebx, incr; // ebx = i;
mov ecx, 0x0; // ecx = 0;
lock xadd dword ptr [eax], ecx;
lock xadd dword ptr [eax], ebx;
mov result, ecx; // result = ebx;
}
return result;
#endif
}
// make sure the static xid is initialized before any threads started
__attribute__((constructor)) int32_t get_xid()
{
static int32_t xid = -1;
if (xid == -1) {
xid = time(0);
}
return fetch_and_add(&xid,1);
}
int enter_critical(zhandle_t* zh)
{
struct adaptor_threads *adaptor = zh->adaptor_priv;
if (adaptor) {
return pthread_mutex_lock(&adaptor->zh_lock);
} else {
return 0;
}
}
int leave_critical(zhandle_t* zh)
{
struct adaptor_threads *adaptor = zh->adaptor_priv;
if (adaptor) {
return pthread_mutex_unlock(&adaptor->zh_lock);
} else {
return 0;
}
}

360
contrib/libzookeeper/src/recordio.c Normal file
View File

@ -0,0 +1,360 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <recordio.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#ifndef WIN32
#include <netinet/in.h>
#endif
void deallocate_String(char **s)
{
if (*s)
free(*s);
*s = 0;
}
void deallocate_Buffer(struct buffer *b)
{
if (b->buff)
free(b->buff);
b->buff = 0;
}
struct buff_struct {
int32_t len;
int32_t off;
char *buffer;
};
static int resize_buffer(struct buff_struct *s, int newlen)
{
char *buffer= NULL;
while (s->len < newlen) {
s->len *= 2;
}
buffer = (char*)realloc(s->buffer, s->len);
if (!buffer) {
s->buffer = 0;
return -ENOMEM;
}
s->buffer = buffer;
return 0;
}
int oa_start_record(struct oarchive *oa, const char *tag)
{
return 0;
}
int oa_end_record(struct oarchive *oa, const char *tag)
{
return 0;
}
int oa_serialize_int(struct oarchive *oa, const char *tag, const int32_t *d)
{
struct buff_struct *priv = oa->priv;
int32_t i = htonl(*d);
if ((priv->len - priv->off) < sizeof(i)) {
int rc = resize_buffer(priv, priv->len + sizeof(i));
if (rc < 0) return rc;
}
memcpy(priv->buffer+priv->off, &i, sizeof(i));
priv->off+=sizeof(i);
return 0;
}
int64_t zoo_htonll(int64_t v)
{
int i = 0;
char *s = (char *)&v;
if (htonl(1) == 1) {
return v;
}
for (i = 0; i < 4; i++) {
int tmp = s[i];
s[i] = s[8-i-1];
s[8-i-1] = tmp;
}
return v;
}
int oa_serialize_long(struct oarchive *oa, const char *tag, const int64_t *d)
{
const int64_t i = zoo_htonll(*d);
struct buff_struct *priv = oa->priv;
if ((priv->len - priv->off) < sizeof(i)) {
int rc = resize_buffer(priv, priv->len + sizeof(i));
if (rc < 0) return rc;
}
memcpy(priv->buffer+priv->off, &i, sizeof(i));
priv->off+=sizeof(i);
return 0;
}
int oa_start_vector(struct oarchive *oa, const char *tag, const int32_t *count)
{
return oa_serialize_int(oa, tag, count);
}
int oa_end_vector(struct oarchive *oa, const char *tag)
{
return 0;
}
int oa_serialize_bool(struct oarchive *oa, const char *name, const int32_t *i)
{
//return oa_serialize_int(oa, name, i);
struct buff_struct *priv = oa->priv;
if ((priv->len - priv->off) < 1) {
int rc = resize_buffer(priv, priv->len + 1);
if (rc < 0)
return rc;
}
priv->buffer[priv->off] = (*i == 0 ? '\0' : '\1');
priv->off++;
return 0;
}
static const int32_t negone = -1;
int oa_serialize_buffer(struct oarchive *oa, const char *name,
const struct buffer *b)
{
struct buff_struct *priv = oa->priv;
int rc;
if (!b) {
return oa_serialize_int(oa, "len", &negone);
}
rc = oa_serialize_int(oa, "len", &b->len);
if (rc < 0)
return rc;
// this means a buffer of NUll
// with size of -1. This is
// waht we use in java serialization for NULL
if (b->len == -1) {
return rc;
}
if ((priv->len - priv->off) < b->len) {
rc = resize_buffer(priv, priv->len + b->len);
if (rc < 0)
return rc;
}
memcpy(priv->buffer+priv->off, b->buff, b->len);
priv->off += b->len;
return 0;
}
int oa_serialize_string(struct oarchive *oa, const char *name, char **s)
{
struct buff_struct *priv = oa->priv;
int32_t len;
int rc;
if (!*s) {
oa_serialize_int(oa, "len", &negone);
return 0;
}
len = strlen(*s);
rc = oa_serialize_int(oa, "len", &len);
if (rc < 0)
return rc;
if ((priv->len - priv->off) < len) {
rc = resize_buffer(priv, priv->len + len);
if (rc < 0)
return rc;
}
memcpy(priv->buffer+priv->off, *s, len);
priv->off += len;
return 0;
}
int ia_start_record(struct iarchive *ia, const char *tag)
{
return 0;
}
int ia_end_record(struct iarchive *ia, const char *tag)
{
return 0;
}
int ia_deserialize_int(struct iarchive *ia, const char *tag, int32_t *count)
{
struct buff_struct *priv = ia->priv;
if ((priv->len - priv->off) < sizeof(*count)) {
return -E2BIG;
}
memcpy(count, priv->buffer+priv->off, sizeof(*count));
priv->off+=sizeof(*count);
*count = ntohl(*count);
return 0;
}
int ia_deserialize_long(struct iarchive *ia, const char *tag, int64_t *count)
{
struct buff_struct *priv = ia->priv;
int64_t v = 0;
if ((priv->len - priv->off) < sizeof(*count)) {
return -E2BIG;
}
memcpy(count, priv->buffer+priv->off, sizeof(*count));
priv->off+=sizeof(*count);
v = zoo_htonll(*count); // htonll and ntohll do the same
*count = v;
return 0;
}
int ia_start_vector(struct iarchive *ia, const char *tag, int32_t *count)
{
return ia_deserialize_int(ia, tag, count);
}
int ia_end_vector(struct iarchive *ia, const char *tag)
{
return 0;
}
int ia_deserialize_bool(struct iarchive *ia, const char *name, int32_t *v)
{
struct buff_struct *priv = ia->priv;
//fprintf(stderr, "Deserializing bool %d\n", priv->off);
//return ia_deserialize_int(ia, name, v);
if ((priv->len - priv->off) < 1) {
return -E2BIG;
}
*v = priv->buffer[priv->off];
priv->off+=1;
//fprintf(stderr, "Deserializing bool end %d\n", priv->off);
return 0;
}
int ia_deserialize_buffer(struct iarchive *ia, const char *name,
struct buffer *b)
{
struct buff_struct *priv = ia->priv;
int rc = ia_deserialize_int(ia, "len", &b->len);
if (rc < 0)
return rc;
if ((priv->len - priv->off) < b->len) {
return -E2BIG;
}
// set the buffer to null
if (b->len == -1) {
b->buff = NULL;
return rc;
}
b->buff = malloc(b->len);
if (!b->buff) {
return -ENOMEM;
}
memcpy(b->buff, priv->buffer+priv->off, b->len);
priv->off += b->len;
return 0;
}
int ia_deserialize_string(struct iarchive *ia, const char *name, char **s)
{
struct buff_struct *priv = ia->priv;
int32_t len;
int rc = ia_deserialize_int(ia, "len", &len);
if (rc < 0)
return rc;
if ((priv->len - priv->off) < len) {
return -E2BIG;
}
if (len < 0) {
return -EINVAL;
}
*s = malloc(len+1);
if (!*s) {
return -ENOMEM;
}
memcpy(*s, priv->buffer+priv->off, len);
(*s)[len] = '\0';
priv->off += len;
return 0;
}
static struct iarchive ia_default = { STRUCT_INITIALIZER (start_record ,ia_start_record),
STRUCT_INITIALIZER (end_record ,ia_end_record), STRUCT_INITIALIZER (start_vector , ia_start_vector),
STRUCT_INITIALIZER (end_vector ,ia_end_vector), STRUCT_INITIALIZER (deserialize_Bool , ia_deserialize_bool),
STRUCT_INITIALIZER (deserialize_Int ,ia_deserialize_int),
STRUCT_INITIALIZER (deserialize_Long , ia_deserialize_long) ,
STRUCT_INITIALIZER (deserialize_Buffer, ia_deserialize_buffer),
STRUCT_INITIALIZER (deserialize_String, ia_deserialize_string) };
static struct oarchive oa_default = { STRUCT_INITIALIZER (start_record , oa_start_record),
STRUCT_INITIALIZER (end_record , oa_end_record), STRUCT_INITIALIZER (start_vector , oa_start_vector),
STRUCT_INITIALIZER (end_vector , oa_end_vector), STRUCT_INITIALIZER (serialize_Bool , oa_serialize_bool),
STRUCT_INITIALIZER (serialize_Int , oa_serialize_int),
STRUCT_INITIALIZER (serialize_Long , oa_serialize_long) ,
STRUCT_INITIALIZER (serialize_Buffer , oa_serialize_buffer),
STRUCT_INITIALIZER (serialize_String , oa_serialize_string) };
struct iarchive *create_buffer_iarchive(char *buffer, int len)
{
struct iarchive *ia = malloc(sizeof(*ia));
struct buff_struct *buff = malloc(sizeof(struct buff_struct));
if (!ia) return 0;
if (!buff) {
free(ia);
return 0;
}
*ia = ia_default;
buff->off = 0;
buff->buffer = buffer;
buff->len = len;
ia->priv = buff;
return ia;
}
struct oarchive *create_buffer_oarchive()
{
struct oarchive *oa = malloc(sizeof(*oa));
struct buff_struct *buff = malloc(sizeof(struct buff_struct));
if (!oa) return 0;
if (!buff) {
free(oa);
return 0;
}
*oa = oa_default;
buff->off = 0;
buff->buffer = malloc(128);
buff->len = 128;
oa->priv = buff;
return oa;
}
void close_buffer_iarchive(struct iarchive **ia)
{
free((*ia)->priv);
free(*ia);
*ia = 0;
}
void close_buffer_oarchive(struct oarchive **oa, int free_buffer)
{
if (free_buffer) {
struct buff_struct *buff = (struct buff_struct *)(*oa)->priv;
if (buff->buffer) {
free(buff->buffer);
}
}
free((*oa)->priv);
free(*oa);
*oa = 0;
}
char *get_buffer(struct oarchive *oa)
{
struct buff_struct *buff = oa->priv;
return buff->buffer;
}
int get_buffer_len(struct oarchive *oa)
{
struct buff_struct *buff = oa->priv;
return buff->off;
}

113
contrib/libzookeeper/src/st_adaptor.c Normal file
View File

@ -0,0 +1,113 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef DLL_EXPORT
# define USE_STATIC_LIB
#endif
#include "zk_adaptor.h"
#include <stdlib.h>
#include <time.h>
int zoo_lock_auth(zhandle_t *zh)
{
return 0;
}
int zoo_unlock_auth(zhandle_t *zh)
{
return 0;
}
int lock_buffer_list(buffer_head_t *l)
{
return 0;
}
int unlock_buffer_list(buffer_head_t *l)
{
return 0;
}
int lock_completion_list(completion_head_t *l)
{
return 0;
}
int unlock_completion_list(completion_head_t *l)
{
return 0;
}
struct sync_completion *alloc_sync_completion(void)
{
return (struct sync_completion*)calloc(1, sizeof(struct sync_completion));
}
int wait_sync_completion(struct sync_completion *sc)
{
return 0;
}
void free_sync_completion(struct sync_completion *sc)
{
free(sc);
}
void notify_sync_completion(struct sync_completion *sc)
{
}
int process_async(int outstanding_sync)
{
return outstanding_sync == 0;
}
int adaptor_init(zhandle_t *zh)
{
return 0;
}
void adaptor_finish(zhandle_t *zh){}
void adaptor_destroy(zhandle_t *zh){}
int flush_send_queue(zhandle_t *, int);
int adaptor_send_queue(zhandle_t *zh, int timeout)
{
return flush_send_queue(zh, timeout);
}
int32_t inc_ref_counter(zhandle_t* zh,int i)
{
zh->ref_counter+=(i<0?-1:(i>0?1:0));
return zh->ref_counter;
}
int32_t get_xid()
{
static int32_t xid = -1;
if (xid == -1) {
xid = time(0);
}
return xid++;
}
int enter_critical(zhandle_t* zh)
{
return 0;
}
int leave_critical(zhandle_t* zh)
{
return 0;
}

276
contrib/libzookeeper/src/zk_adaptor.h Normal file
View File

@ -0,0 +1,276 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ZK_ADAPTOR_H_
#define ZK_ADAPTOR_H_
#include <zookeeper.jute.h>
#ifdef THREADED
#ifndef WIN32
#include <pthread.h>
#else
#include "winport.h"
#endif
#endif
#include "zookeeper.h"
#include "zk_hashtable.h"
/* predefined xid's values recognized as special by the server */
#define WATCHER_EVENT_XID -1
#define PING_XID -2
#define AUTH_XID -4
#define SET_WATCHES_XID -8
/* zookeeper state constants */
#define EXPIRED_SESSION_STATE_DEF -112
#define AUTH_FAILED_STATE_DEF -113
#define CONNECTING_STATE_DEF 1
#define ASSOCIATING_STATE_DEF 2
#define CONNECTED_STATE_DEF 3
#define NOTCONNECTED_STATE_DEF 999
/* zookeeper event type constants */
#define CREATED_EVENT_DEF 1
#define DELETED_EVENT_DEF 2
#define CHANGED_EVENT_DEF 3
#define CHILD_EVENT_DEF 4
#define SESSION_EVENT_DEF -1
#define NOTWATCHING_EVENT_DEF -2
#ifdef __cplusplus
extern "C" {
#endif
struct _buffer_list;
struct _completion_list;
typedef struct _buffer_head {
struct _buffer_list *volatile head;
struct _buffer_list *last;
#ifdef THREADED
pthread_mutex_t lock;
#endif
} buffer_head_t;
typedef struct _completion_head {
struct _completion_list *volatile head;
struct _completion_list *last;
#ifdef THREADED
pthread_cond_t cond;
pthread_mutex_t lock;
#endif
} completion_head_t;
int lock_buffer_list(buffer_head_t *l);
int unlock_buffer_list(buffer_head_t *l);
int lock_completion_list(completion_head_t *l);
int unlock_completion_list(completion_head_t *l);
struct sync_completion {
int rc;
union {
struct {
char *str;
int str_len;
} str;
struct Stat stat;
struct {
char *buffer;
int buff_len;
struct Stat stat;
} data;
struct {
struct ACL_vector acl;
struct Stat stat;
} acl;
struct String_vector strs2;
struct {
struct String_vector strs2;
struct Stat stat2;
} strs_stat;
} u;
int complete;
#ifdef THREADED
pthread_cond_t cond;
pthread_mutex_t lock;
#endif
};
typedef struct _auth_info {
int state; /* 0=>inactive, >0 => active */
char* scheme;
struct buffer auth;
void_completion_t completion;
const char* data;
struct _auth_info *next;
} auth_info;
/**
* This structure represents a packet being read or written.
*/
typedef struct _buffer_list {
char *buffer;
int len; /* This represents the length of sizeof(header) + length of buffer */
int curr_offset; /* This is the offset into the header followed by offset into the buffer */
struct _buffer_list *next;
} buffer_list_t;
/* the size of connect request */
#define HANDSHAKE_REQ_SIZE 44
/* connect request */
struct connect_req {
int32_t protocolVersion;
int64_t lastZxidSeen;
int32_t timeOut;
int64_t sessionId;
int32_t passwd_len;
char passwd[16];
};
/* the connect response */
struct prime_struct {
int32_t len;
int32_t protocolVersion;
int32_t timeOut;
int64_t sessionId;
int32_t passwd_len;
char passwd[16];
};
#ifdef THREADED
/* this is used by mt_adaptor internally for thread management */
struct adaptor_threads {
pthread_t io;
pthread_t completion;
int threadsToWait; // barrier
pthread_cond_t cond; // barrier's conditional
pthread_mutex_t lock; // ... and a lock
pthread_mutex_t zh_lock; // critical section lock
#ifdef WIN32
SOCKET self_pipe[2];
#else
int self_pipe[2];
#endif
};
#endif
/** the auth list for adding auth */
typedef struct _auth_list_head {
auth_info *auth;
#ifdef THREADED
pthread_mutex_t lock;
#endif
} auth_list_head_t;
/**
* This structure represents the connection to zookeeper.
*/
struct _zhandle {
#ifdef WIN32
SOCKET fd; /* the descriptor used to talk to zookeeper */
#else
int fd; /* the descriptor used to talk to zookeeper */
#endif
char *hostname; /* the hostname of zookeeper */
struct sockaddr_storage *addrs; /* the addresses that correspond to the hostname */
int addrs_count; /* The number of addresses in the addrs array */
watcher_fn watcher; /* the registered watcher */
struct timeval last_recv; /* The time that the last message was received */
struct timeval last_send; /* The time that the last message was sent */
struct timeval last_ping; /* The time that the last PING was sent */
struct timeval next_deadline; /* The time of the next deadline */
int recv_timeout; /* The maximum amount of time that can go by without
receiving anything from the zookeeper server */
buffer_list_t *input_buffer; /* the current buffer being read in */
buffer_head_t to_process; /* The buffers that have been read and are ready to be processed. */
buffer_head_t to_send; /* The packets queued to send */
completion_head_t sent_requests; /* The outstanding requests */
completion_head_t completions_to_process; /* completions that are ready to run */
int connect_index; /* The index of the address to connect to */
clientid_t client_id;
long long last_zxid;
int outstanding_sync; /* Number of outstanding synchronous requests */
struct _buffer_list primer_buffer; /* The buffer used for the handshake at the start of a connection */
struct prime_struct primer_storage; /* the connect response */
char primer_storage_buffer[40]; /* the true size of primer_storage */
volatile int state;
void *context;
auth_list_head_t auth_h; /* authentication data list */
/* zookeeper_close is not reentrant because it de-allocates the zhandler.
* This guard variable is used to defer the destruction of zhandle till
* right before top-level API call returns to the caller */
int32_t ref_counter;
volatile int close_requested;
void *adaptor_priv;
/* Used for debugging only: non-zero value indicates the time when the zookeeper_process
* call returned while there was at least one unprocessed server response
* available in the socket recv buffer */
struct timeval socket_readable;
zk_hashtable* active_node_watchers;
zk_hashtable* active_exist_watchers;
zk_hashtable* active_child_watchers;
/** used for chroot path at the client side **/
char *chroot;
};
int adaptor_init(zhandle_t *zh);
void adaptor_finish(zhandle_t *zh);
void adaptor_destroy(zhandle_t *zh);
struct sync_completion *alloc_sync_completion(void);
int wait_sync_completion(struct sync_completion *sc);
void free_sync_completion(struct sync_completion *sc);
void notify_sync_completion(struct sync_completion *sc);
int adaptor_send_queue(zhandle_t *zh, int timeout);
int process_async(int outstanding_sync);
void process_completions(zhandle_t *zh);
int flush_send_queue(zhandle_t*zh, int timeout);
char* sub_string(zhandle_t *zh, const char* server_path);
void free_duplicate_path(const char* free_path, const char* path);
int zoo_lock_auth(zhandle_t *zh);
int zoo_unlock_auth(zhandle_t *zh);
// critical section guards
int enter_critical(zhandle_t* zh);
int leave_critical(zhandle_t* zh);
// zhandle object reference counting
void api_prolog(zhandle_t* zh);
int api_epilog(zhandle_t *zh, int rc);
int32_t get_xid();
// returns the new value of the ref counter
int32_t inc_ref_counter(zhandle_t* zh,int i);
#ifdef THREADED
// atomic post-increment
int32_t fetch_and_add(volatile int32_t* operand, int incr);
// in mt mode process session event asynchronously by the completion thread
#define PROCESS_SESSION_EVENT(zh,newstate) queue_session_event(zh,newstate)
#else
// in single-threaded mode process session event immediately
//#define PROCESS_SESSION_EVENT(zh,newstate) deliverWatchers(zh,ZOO_SESSION_EVENT,newstate,0)
#define PROCESS_SESSION_EVENT(zh,newstate) queue_session_event(zh,newstate)
#endif
#ifdef __cplusplus
}
#endif
#endif /*ZK_ADAPTOR_H_*/

337
contrib/libzookeeper/src/zk_hashtable.c Normal file
View File

@ -0,0 +1,337 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "zk_hashtable.h"
#include "zk_adaptor.h"
#include "hashtable/hashtable.h"
#include "hashtable/hashtable_itr.h"
#include <string.h>
#include <stdlib.h>
#include <assert.h>
typedef struct _watcher_object {
watcher_fn watcher;
void* context;
struct _watcher_object* next;
} watcher_object_t;
struct _zk_hashtable {
struct hashtable* ht;
};
struct watcher_object_list {
watcher_object_t* head;
};
/* the following functions are for testing only */
typedef struct hashtable hashtable_impl;
hashtable_impl* getImpl(zk_hashtable* ht){
return ht->ht;
}
watcher_object_t* getFirstWatcher(zk_hashtable* ht,const char* path)
{
watcher_object_list_t* wl=hashtable_search(ht->ht,(void*)path);
if(wl!=0)
return wl->head;
return 0;
}
/* end of testing functions */
watcher_object_t* clone_watcher_object(watcher_object_t* wo)
{
watcher_object_t* res=calloc(1,sizeof(watcher_object_t));
assert(res);
res->watcher=wo->watcher;
res->context=wo->context;
return res;
}
static unsigned int string_hash_djb2(void *str)
{
unsigned int hash = 5381;
int c;
const char* cstr = (const char*)str;
while ((c = *cstr++))
hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
return hash;
}
static int string_equal(void *key1,void *key2)
{
return strcmp((const char*)key1,(const char*)key2)==0;
}
static watcher_object_t* create_watcher_object(watcher_fn watcher,void* ctx)
{
watcher_object_t* wo=calloc(1,sizeof(watcher_object_t));
assert(wo);
wo->watcher=watcher;
wo->context=ctx;
return wo;
}
static watcher_object_list_t* create_watcher_object_list(watcher_object_t* head)
{
watcher_object_list_t* wl=calloc(1,sizeof(watcher_object_list_t));
assert(wl);
wl->head=head;
return wl;
}
static void destroy_watcher_object_list(watcher_object_list_t* list)
{
watcher_object_t* e = NULL;
if(list==0)
return;
e=list->head;
while(e!=0){
watcher_object_t* this=e;
e=e->next;
free(this);
}
free(list);
}
zk_hashtable* create_zk_hashtable()
{
struct _zk_hashtable *ht=calloc(1,sizeof(struct _zk_hashtable));
assert(ht);
ht->ht=create_hashtable(32,string_hash_djb2,string_equal);
return ht;
}
static void do_clean_hashtable(zk_hashtable* ht)
{
struct hashtable_itr *it;
int hasMore;
if(hashtable_count(ht->ht)==0)
return;
it=hashtable_iterator(ht->ht);
do {
watcher_object_list_t* w=hashtable_iterator_value(it);
destroy_watcher_object_list(w);
hasMore=hashtable_iterator_remove(it);
} while(hasMore);
free(it);
}
void destroy_zk_hashtable(zk_hashtable* ht)
{
if(ht!=0){
do_clean_hashtable(ht);
hashtable_destroy(ht->ht,0);
free(ht);
}
}
// searches for a watcher object instance in a watcher object list;
// two watcher objects are equal if their watcher function and context pointers
// are equal
static watcher_object_t* search_watcher(watcher_object_list_t** wl,watcher_object_t* wo)
{
watcher_object_t* wobj=(*wl)->head;
while(wobj!=0){
if(wobj->watcher==wo->watcher && wobj->context==wo->context)
return wobj;
wobj=wobj->next;
}
return 0;
}
static int add_to_list(watcher_object_list_t **wl, watcher_object_t *wo,
int clone)
{
if (search_watcher(wl, wo)==0) {
watcher_object_t* cloned=wo;
if (clone) {
cloned = clone_watcher_object(wo);
assert(cloned);
}
cloned->next = (*wl)->head;
(*wl)->head = cloned;
return 1;
} else if (!clone) {
// If it's here and we aren't supposed to clone, we must destroy
free(wo);
}
return 0;
}
static int do_insert_watcher_object(zk_hashtable *ht, const char *path, watcher_object_t* wo)
{
int res=1;
watcher_object_list_t* wl;
wl=hashtable_search(ht->ht,(void*)path);
if(wl==0){
int res;
/* inserting a new path element */
res=hashtable_insert(ht->ht,strdup(path),create_watcher_object_list(wo));
assert(res);
}else{
/*
* Path already exists; check if the watcher already exists.
* Don't clone the watcher since it's allocated on the heap --- avoids
* a memory leak and saves a clone operation (calloc + copy).
*/
res = add_to_list(&wl, wo, 0);
}
return res;
}
char **collect_keys(zk_hashtable *ht, int *count)
{
char **list;
struct hashtable_itr *it;
int i;
*count = hashtable_count(ht->ht);
list = calloc(*count, sizeof(char*));
it=hashtable_iterator(ht->ht);
for(i = 0; i < *count; i++) {
list[i] = strdup(hashtable_iterator_key(it));
hashtable_iterator_advance(it);
}
free(it);
return list;
}
static int insert_watcher_object(zk_hashtable *ht, const char *path,
watcher_object_t* wo)
{
int res;
res=do_insert_watcher_object(ht,path,wo);
return res;
}
static void copy_watchers(watcher_object_list_t *from, watcher_object_list_t *to, int clone)
{
watcher_object_t* wo=from->head;
while(wo){
watcher_object_t *next = wo->next;
add_to_list(&to, wo, clone);
wo=next;
}
}
static void copy_table(zk_hashtable *from, watcher_object_list_t *to) {
struct hashtable_itr *it;
int hasMore;
if(hashtable_count(from->ht)==0)
return;
it=hashtable_iterator(from->ht);
do {
watcher_object_list_t *w = hashtable_iterator_value(it);
copy_watchers(w, to, 1);
hasMore=hashtable_iterator_advance(it);
} while(hasMore);
free(it);
}
static void collect_session_watchers(zhandle_t *zh,
watcher_object_list_t **list)
{
copy_table(zh->active_node_watchers, *list);
copy_table(zh->active_exist_watchers, *list);
copy_table(zh->active_child_watchers, *list);
}
static void add_for_event(zk_hashtable *ht, char *path, watcher_object_list_t **list)
{
watcher_object_list_t* wl;
wl = (watcher_object_list_t*)hashtable_remove(ht->ht, path);
if (wl) {
copy_watchers(wl, *list, 0);
// Since we move, not clone the watch_objects, we just need to free the
// head pointer
free(wl);
}
}
static void do_foreach_watcher(watcher_object_t* wo,zhandle_t* zh,
const char* path,int type,int state)
{
// session event's don't have paths
const char *client_path =
(type != ZOO_SESSION_EVENT ? sub_string(zh, path) : path);
while(wo!=0){
wo->watcher(zh,type,state,client_path,wo->context);
wo=wo->next;
}
free_duplicate_path(client_path, path);
}
watcher_object_list_t *collectWatchers(zhandle_t *zh,int type, char *path)
{
struct watcher_object_list *list = create_watcher_object_list(0);
if(type==ZOO_SESSION_EVENT){
watcher_object_t defWatcher;
defWatcher.watcher=zh->watcher;
defWatcher.context=zh->context;
add_to_list(&list, &defWatcher, 1);
collect_session_watchers(zh, &list);
return list;
}
switch(type){
case CREATED_EVENT_DEF:
case CHANGED_EVENT_DEF:
// look up the watchers for the path and move them to a delivery list
add_for_event(zh->active_node_watchers,path,&list);
add_for_event(zh->active_exist_watchers,path,&list);
break;
case CHILD_EVENT_DEF:
// look up the watchers for the path and move them to a delivery list
add_for_event(zh->active_child_watchers,path,&list);
break;
case DELETED_EVENT_DEF:
// look up the watchers for the path and move them to a delivery list
add_for_event(zh->active_node_watchers,path,&list);
add_for_event(zh->active_exist_watchers,path,&list);
add_for_event(zh->active_child_watchers,path,&list);
break;
}
return list;
}
void deliverWatchers(zhandle_t *zh, int type,int state, char *path, watcher_object_list_t **list)
{
if (!list || !(*list)) return;
do_foreach_watcher((*list)->head, zh, path, type, state);
destroy_watcher_object_list(*list);
*list = 0;
}
void activateWatcher(zhandle_t *zh, watcher_registration_t* reg, int rc)
{
if(reg){
/* in multithreaded lib, this code is executed
* by the IO thread */
zk_hashtable *ht = reg->checker(zh, rc);
if(ht){
insert_watcher_object(ht,reg->path,
create_watcher_object(reg->watcher, reg->context));
}
}
}

69
contrib/libzookeeper/src/zk_hashtable.h Normal file
View File

@ -0,0 +1,69 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ZK_HASHTABLE_H_
#define ZK_HASHTABLE_H_
#include <zookeeper.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct watcher_object_list watcher_object_list_t;
typedef struct _zk_hashtable zk_hashtable;
/**
* The function must return a non-zero value if the watcher object can be activated
* as a result of the server response. Normally, a watch can only be activated
* if the server returns a success code (ZOK). However in the case when zoo_exists()
* returns a ZNONODE code the watcher should be activated nevertheless.
*/
typedef zk_hashtable *(*result_checker_fn)(zhandle_t *, int rc);
/**
* A watcher object gets temporarily stored with the completion entry until
* the server response comes back at which moment the watcher object is moved
* to the active watchers map.
*/
typedef struct _watcher_registration {
watcher_fn watcher;
void* context;
result_checker_fn checker;
const char* path;
} watcher_registration_t;
zk_hashtable* create_zk_hashtable();
void destroy_zk_hashtable(zk_hashtable* ht);
char **collect_keys(zk_hashtable *ht, int *count);
/**
* check if the completion has a watcher object associated
* with it. If it does, move the watcher object to the map of
* active watchers (only if the checker allows to do so)
*/
void activateWatcher(zhandle_t *zh, watcher_registration_t* reg, int rc);
watcher_object_list_t *collectWatchers(zhandle_t *zh,int type, char *path);
void deliverWatchers(zhandle_t *zh, int type, int state, char *path, struct watcher_object_list **list);
#ifdef __cplusplus
}
#endif
#endif /*ZK_HASHTABLE_H_*/

177
contrib/libzookeeper/src/zk_log.c Normal file
View File

@ -0,0 +1,177 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef DLL_EXPORT
# define USE_STATIC_LIB
#endif
#include "zookeeper_log.h"
#ifndef WIN32
#include <unistd.h>
#endif
#include <stdarg.h>
#include <time.h>
#define TIME_NOW_BUF_SIZE 1024
#define FORMAT_LOG_BUF_SIZE 4096
#ifdef THREADED
#ifndef WIN32
#include <pthread.h>
#else
#include "winport.h"
#endif
static pthread_key_t time_now_buffer;
static pthread_key_t format_log_msg_buffer;
void freeBuffer(void* p){
if(p) free(p);
}
__attribute__((constructor)) void prepareTSDKeys() {
pthread_key_create (&time_now_buffer, freeBuffer);
pthread_key_create (&format_log_msg_buffer, freeBuffer);
}
char* getTSData(pthread_key_t key,int size){
char* p=pthread_getspecific(key);
if(p==0){
int res;
p=calloc(1,size);
res=pthread_setspecific(key,p);
if(res!=0){
fprintf(stderr,"Failed to set TSD key: %d",res);
}
}
return p;
}
char* get_time_buffer(){
return getTSData(time_now_buffer,TIME_NOW_BUF_SIZE);
}
char* get_format_log_buffer(){
return getTSData(format_log_msg_buffer,FORMAT_LOG_BUF_SIZE);
}
#else
char* get_time_buffer(){
static char buf[TIME_NOW_BUF_SIZE];
return buf;
}
char* get_format_log_buffer(){
static char buf[FORMAT_LOG_BUF_SIZE];
return buf;
}
#endif
ZooLogLevel logLevel=ZOO_LOG_LEVEL_INFO;
static FILE* logStream=0;
FILE* getLogStream(){
if(logStream==0)
logStream=stderr;
return logStream;
}
void zoo_set_log_stream(FILE* stream){
logStream=stream;
}
static const char* time_now(char* now_str){
struct timeval tv;
struct tm lt;
time_t now = 0;
size_t len = 0;
gettimeofday(&tv,0);
now = tv.tv_sec;
localtime_r(&now, &lt);
// clone the format used by log4j ISO8601DateFormat
// specifically: "yyyy-MM-dd HH:mm:ss,SSS"
len = strftime(now_str, TIME_NOW_BUF_SIZE,
"%Y-%m-%d %H:%M:%S",
&lt);
len += snprintf(now_str + len,
TIME_NOW_BUF_SIZE - len,
",%03d",
(int)(tv.tv_usec/1000));
return now_str;
}
void log_message(ZooLogLevel curLevel,int line,const char* funcName,
const char* message)
{
static const char* dbgLevelStr[]={"ZOO_INVALID","ZOO_ERROR","ZOO_WARN",
"ZOO_INFO","ZOO_DEBUG"};
static pid_t pid=0;
#ifdef WIN32
char timebuf [TIME_NOW_BUF_SIZE];
#endif
if(pid==0)pid=getpid();
#ifndef THREADED
// pid_t is long on Solaris
fprintf(LOGSTREAM, "%s:%ld:%s@%s@%d: %s\n", time_now(get_time_buffer()),(long)pid,
dbgLevelStr[curLevel],funcName,line,message);
#else
#ifdef WIN32
fprintf(LOGSTREAM, "%s:%d(0x%lx):%s@%s@%d: %s\n", time_now(timebuf),pid,
(unsigned long int)(pthread_self().thread_id),
dbgLevelStr[curLevel],funcName,line,message);
#else
fprintf(LOGSTREAM, "%s:%ld(0x%lx):%s@%s@%d: %s\n", time_now(get_time_buffer()),(long)pid,
(unsigned long int)pthread_self(),
dbgLevelStr[curLevel],funcName,line,message);
#endif
#endif
fflush(LOGSTREAM);
}
const char* format_log_message(const char* format,...)
{
va_list va;
char* buf=get_format_log_buffer();
if(!buf)
return "format_log_message: Unable to allocate memory buffer";
va_start(va,format);
vsnprintf(buf, FORMAT_LOG_BUF_SIZE-1,format,va);
va_end(va);
return buf;
}
void zoo_set_debug_level(ZooLogLevel level)
{
if(level==0){
// disable logging (unit tests do this)
logLevel=(ZooLogLevel)0;
return;
}
if(level<ZOO_LOG_LEVEL_ERROR)level=ZOO_LOG_LEVEL_ERROR;
if(level>ZOO_LOG_LEVEL_DEBUG)level=ZOO_LOG_LEVEL_DEBUG;
logLevel=level;
}

3729
contrib/libzookeeper/src/zookeeper.c Normal file
View File

File diff suppressed because it is too large Load Diff

1315
contrib/libzookeeper/src/zookeeper.jute.c Normal file
View File

File diff suppressed because it is too large Load Diff

38
dbms/CMakeLists.txt
View File

@ -12,18 +12,19 @@ endif()
if (${DISABLE_MONGODB})
add_definitions(-D DISABLE_MONGODB)
else()
set (LINK_MONGOCLIENT libmongoclient.a libssl.a libcrypto.a)
set (LINK_MONGOCLIENT libmongoclient.a libssl.a libcrypto.a libboost_thread.a)
endif()
add_library(string_utils
include/DB/Common/StringUtils.h
src/Common/StringUtils.cpp)
add_library (dbms
src/Server/OLAPAttributesMetadata.h
src/Server/InterserverIOHTTPHandler.h
src/Server/OLAPHTTPHandler.h
src/Server/OLAPQueryConverter.h
src/Server/Server.h
src/Server/TCPHandler.h
src/Server/HTTPHandler.h
src/Server/OLAPQueryParser.h
src/Server/MetricsTransmitter.h
src/Server/UsersConfigReloader.h
src/Server/StatusFile.h
@ -60,6 +61,7 @@ add_library (dbms
include/DB/Functions/FunctionsComparison.h
include/DB/Functions/FunctionsHashing.h
include/DB/Functions/FunctionsMath.h
include/DB/Functions/FunctionsGeo.h
include/DB/Functions/FunctionsMiscellaneous.h
include/DB/Functions/FunctionsDateTime.h
include/DB/Functions/IFunction.h
@ -82,7 +84,6 @@ add_library (dbms
include/DB/Parsers/ASTExpressionList.h
include/DB/Parsers/ASTQueryWithOutput.h
include/DB/Parsers/ParserSelectQuery.h
include/DB/Parsers/ParserTableExpression.h
include/DB/Parsers/ParserUseQuery.h
include/DB/Parsers/ASTShowTablesQuery.h
include/DB/Parsers/ASTFunction.h
@ -95,9 +96,7 @@ add_library (dbms
include/DB/Parsers/ASTSubquery.h
include/DB/Parsers/ASTUseQuery.h
include/DB/Parsers/ASTIdentifier.h
include/DB/Parsers/ParserJoin.h
include/DB/Parsers/ParserTablePropertiesQuery.h
include/DB/Parsers/ASTJoin.h
include/DB/Parsers/ParserCheckQuery.h
include/DB/Parsers/ParserRenameQuery.h
include/DB/Parsers/ParserInsertQuery.h
@ -132,6 +131,8 @@ add_library (dbms
include/DB/Parsers/ASTSampleRatio.h
include/DB/Parsers/ParserSampleRatio.h
include/DB/Parsers/ParserCase.h
include/DB/Parsers/ASTTablesInSelectQuery.h
include/DB/Parsers/ParserTablesInSelectQuery.h
include/DB/AggregateFunctions/AggregateFunctionMerge.h
include/DB/AggregateFunctions/AggregateFunctionUniqUpTo.h
include/DB/AggregateFunctions/AggregateFunctionIf.h
@ -265,6 +266,10 @@ add_library (dbms
include/DB/DataStreams/MarkInCompressedFile.h
include/DB/DataStreams/CSVRowOutputStream.h
include/DB/DataStreams/CSVRowInputStream.h
include/DB/DataStreams/verbosePrintString.h
include/DB/DataStreams/SquashingTransform.h
include/DB/DataStreams/SquashingBlockInputStream.h
include/DB/DataStreams/SquashingBlockOutputStream.h
include/DB/DataTypes/IDataType.h
include/DB/DataTypes/IDataTypeDummy.h
include/DB/DataTypes/DataTypeSet.h
@ -428,6 +433,9 @@ add_library (dbms
include/DB/Common/getNumberOfPhysicalCPUCores.h
include/DB/Common/BitHelpers.h
include/DB/Common/BlockFilterCreator.h
include/DB/Common/randomSeed.h
include/DB/Common/unaligned.h
include/DB/Common/ThreadPool.h
include/DB/IO/CompressedStream.h
include/DB/IO/ReadBufferFromFileDescriptor.h
include/DB/IO/CompressedWriteBuffer.h
@ -597,6 +605,8 @@ add_library (dbms
src/Common/ShellCommand.cpp
src/Common/isLocalAddress.cpp
src/Common/getNumberOfPhysicalCPUCores.cpp
src/Common/randomSeed.cpp
src/Common/ThreadPool.cpp
src/Core/Field.cpp
src/Core/FieldVisitors.cpp
@ -742,6 +752,10 @@ add_library (dbms
src/DataStreams/DistinctBlockInputStream.cpp
src/DataStreams/RemoteBlockInputStream.cpp
src/DataStreams/BlockIO.cpp
src/DataStreams/verbosePrintString.cpp
src/DataStreams/SquashingTransform.cpp
src/DataStreams/SquashingBlockInputStream.cpp
src/DataStreams/SquashingBlockOutputStream.cpp
src/DataTypes/DataTypeString.cpp
src/DataTypes/DataTypeFixedString.cpp
@ -771,6 +785,7 @@ add_library (dbms
src/Parsers/ASTWithAlias.cpp
src/Parsers/ASTIdentifier.cpp
src/Parsers/ASTSampleRatio.cpp
src/Parsers/ASTTablesInSelectQuery.cpp
src/Parsers/IAST.cpp
src/Parsers/IParserBase.cpp
src/Parsers/ExpressionElementParsers.cpp
@ -778,8 +793,6 @@ add_library (dbms
src/Parsers/ParserQueryWithOutput.cpp
src/Parsers/ParserCreateQuery.cpp
src/Parsers/ParserSelectQuery.cpp
src/Parsers/ParserTableExpression.cpp
src/Parsers/ParserJoin.cpp
src/Parsers/ParserInsertQuery.cpp
src/Parsers/ParserDropQuery.cpp
src/Parsers/ParserRenameQuery.cpp
@ -792,6 +805,7 @@ add_library (dbms
src/Parsers/ParserCheckQuery.cpp
src/Parsers/ParserSampleRatio.cpp
src/Parsers/ParserCase.cpp
src/Parsers/ParserTablesInSelectQuery.cpp
src/Parsers/formatAST.cpp
src/Parsers/parseQuery.cpp
src/Parsers/queryToString.cpp
@ -861,6 +875,7 @@ add_library (dbms
src/Functions/FunctionsURL.cpp
src/Functions/FunctionsVisitParam.cpp
src/Functions/FunctionsMath.cpp
src/Functions/FunctionsGeo.cpp
src/Functions/FunctionsMiscellaneous.cpp
src/Functions/FunctionsTransform.cpp
src/Functions/Conditional/getArrayType.cpp
@ -926,6 +941,7 @@ SET_SOURCE_FILES_PROPERTIES(
src/Functions/FunctionsURL.cpp
src/Functions/FunctionsVisitParam.cpp
src/Functions/FunctionsMath.cpp
src/Functions/FunctionsGeo.cpp
src/Functions/FunctionsMiscellaneous.cpp
src/Functions/FunctionsTransform.cpp
src/Dictionaries/FlatDictionary.cpp
@ -946,11 +962,11 @@ target_link_libraries(dbms
mysqlxx
cityhash farmhash metrohash
lz4 zstd
string_utils
double-conversion
${LINK_LIBRARIES_ONLY_ON_X86_64}
re2 re2_st
libcrypto.a
libboost_thread.a
libboost_system.a
${LINK_MONGOCLIENT}
libboost_regex.a

6
dbms/include/DB/AggregateFunctions/AggregateFunctionArray.h
View File

@ -19,7 +19,7 @@ class AggregateFunctionArray final : public IAggregateFunction
private:
AggregateFunctionPtr nested_func_owner;
IAggregateFunction * nested_func;
int num_agruments;
size_t num_agruments;
public:
AggregateFunctionArray(AggregateFunctionPtr nested_) : nested_func_owner(nested_), nested_func(nested_func_owner.get()) {}
@ -42,7 +42,7 @@ public:
throw Exception("Array aggregate functions requires at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
DataTypes nested_arguments;
for (int i = 0; i < num_agruments; ++i)
for (size_t i = 0; i < num_agruments; ++i)
{
if (const DataTypeArray * array = typeid_cast<const DataTypeArray *>(&*arguments[i]))
nested_arguments.push_back(array->getNestedType());
@ -87,7 +87,7 @@ public:
{
const IColumn * nested[num_agruments];
for (int i = 0; i < num_agruments; ++i)
for (size_t i = 0; i < num_agruments; ++i)
nested[i] = &static_cast<const ColumnArray &>(*columns[i]).getData();
const ColumnArray & first_array_column = static_cast<const ColumnArray &>(*columns[0]);

54
dbms/include/DB/AggregateFunctions/AggregateFunctionFactory.h
View File

@ -7,13 +7,16 @@
namespace DB
{
/** Позволяет создать агрегатную функцию по её имени.
/** Creates aggregate function by name.
*/
class AggregateFunctionFactory final
{
friend class StorageSystemFunctions;
private:
/// Не std::function, так как меньше indirection и размер объекта.
using Creator = AggregateFunctionPtr(*)(const std::string & name, const DataTypes & argument_types);
/// Not std::function, for lower object size and less indirection.
using Creator = AggregateFunctionPtr(*)(const String & name, const DataTypes & argument_types);
using AggregateFunctions = std::unordered_map<String, Creator>;
public:
AggregateFunctionFactory();
@ -21,45 +24,24 @@ public:
AggregateFunctionPtr tryGet(const String & name, const DataTypes & argument_types) const;
bool isAggregateFunctionName(const String & name, int recursion_level = 0) const;
/// Зарегистрировать агрегатную функцию заданную по одному или нескольким названиям.
void registerFunction(const std::vector<std::string> & names, Creator creator);
/// For compatibility with SQL, it's possible to specify that certain aggregate function name is case insensitive.
enum CaseSensitiveness
{
CaseSensitive,
CaseInsensitive
};
/// Register aggregate function with its name.
void registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
AggregateFunctionFactory(const AggregateFunctionFactory &) = delete;
AggregateFunctionFactory & operator=(const AggregateFunctionFactory &) = delete;
private:
struct Descriptor
{
Creator creator;
bool is_alias;
};
using AggregateFunctions = std::unordered_map<std::string, Descriptor>;
public:
struct Details
{
std::string name;
bool is_alias;
};
private:
/// Вспомогательная функция для реализации итератора (см. ниже).
static Details getDetails(const AggregateFunctions::value_type & entry);
public:
/** Итератор над агрегатными функциями. Возвращает объект типа Details.
* Этот итератор нужен для таблицы system.functions.
*/
using const_iterator = boost::transform_iterator<decltype(&AggregateFunctionFactory::getDetails),
typename AggregateFunctions::const_iterator>;
public:
const_iterator begin() const;
const_iterator end() const;
private:
AggregateFunctions aggregate_functions;
/// Case insensitive aggregate functions will be additionally added here with lowercased name.
AggregateFunctions case_insensitive_aggregate_functions;
};
}

3
dbms/include/DB/AggregateFunctions/AggregateFunctionGroupUniqArray.h
View File

@ -36,7 +36,8 @@ struct AggregateFunctionGroupUniqArrayData
/// Складывает все значения в хэш-множество. Возвращает массив уникальных значений. Реализована для числовых типов.
template <typename T>
class AggregateFunctionGroupUniqArray : public IUnaryAggregateFunction<AggregateFunctionGroupUniqArrayData<T>, AggregateFunctionGroupUniqArray<T> >
class AggregateFunctionGroupUniqArray
: public IUnaryAggregateFunction<AggregateFunctionGroupUniqArrayData<T>, AggregateFunctionGroupUniqArray<T>>
{
private:
using State = AggregateFunctionGroupUniqArrayData<T>;

4
dbms/include/DB/AggregateFunctions/AggregateFunctionIf.h
View File

@ -19,7 +19,7 @@ class AggregateFunctionIf final : public IAggregateFunction
private:
AggregateFunctionPtr nested_func_owner;
IAggregateFunction * nested_func;
int num_agruments;
size_t num_agruments;
public:
AggregateFunctionIf(AggregateFunctionPtr nested_) : nested_func_owner(nested_), nested_func(nested_func_owner.get()) {}
@ -42,7 +42,7 @@ public:
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
DataTypes nested_arguments;
for (int i = 0; i < num_agruments - 1; i ++)
for (size_t i = 0; i < num_agruments - 1; i ++)
nested_arguments.push_back(arguments[i]);
nested_func->setArguments(nested_arguments);
}

3
dbms/include/DB/AggregateFunctions/AggregateFunctionQuantileTiming.h
View File

@ -479,7 +479,7 @@ namespace detail
/** sizeof - 64 байта.
* Если их не хватает - выделяет дополнительно около 20 КБ памяти.
* Если их не хватает - выделяет дополнительно до 20 КБ памяти.
*/
class QuantileTiming : private boost::noncopyable
{
@ -525,6 +525,7 @@ private:
for (const auto & elem : medium.elems)
tmp_large->insert(elem);
medium.~QuantileTimingMedium();
large = tmp_large;
tiny.count = TINY_MAX_ELEMS + 2;
}

20
dbms/include/DB/AggregateFunctions/AggregateFunctionSequenceMatch.h
View File

@ -8,6 +8,7 @@
#include <boost/range/iterator_range_core.hpp>
#include <DB/Parsers/ExpressionElementParsers.h>
#include <DB/Parsers/ASTLiteral.h>
#include <DB/Common/PODArray.h>
#include <bitset>
#include <stack>
@ -44,7 +45,8 @@ struct AggregateFunctionSequenceMatchData final
using Comparator = ComparePairFirst<std::less>;
bool sorted = true;
std::vector<TimestampEvents> eventsList;
static constexpr size_t bytes_in_arena = 64;
PODArray<TimestampEvents, bytes_in_arena, AllocatorWithStackMemory<Allocator<false>, bytes_in_arena>> eventsList;
void add(const Timestamp timestamp, const Events & events)
{
@ -60,7 +62,7 @@ struct AggregateFunctionSequenceMatchData final
{
const auto size = eventsList.size();
eventsList.insert(std::end(eventsList), std::begin(other.eventsList), std::end(other.eventsList));
eventsList.insert(std::begin(other.eventsList), std::end(other.eventsList));
/// either sort whole container or do so partially merging ranges afterwards
if (!sorted && !other.sorted)
@ -111,7 +113,7 @@ struct AggregateFunctionSequenceMatchData final
std::size_t size;
readBinary(size, buf);
decltype(eventsList) eventsList;
eventsList.clear();
eventsList.reserve(size);
for (std::size_t i = 0; i < size; ++i)
@ -124,8 +126,6 @@ struct AggregateFunctionSequenceMatchData final
eventsList.emplace_back(timestamp, Events{events});
}
this->eventsList = std::move(eventsList);
}
};
@ -262,14 +262,14 @@ private:
PatternAction(const PatternActionType type, const std::uint32_t extra = 0) : type{type}, extra{extra} {}
};
using PatternActions = std::vector<PatternAction>;
static constexpr size_t bytes_on_stack = 64;
using PatternActions = PODArray<PatternAction, bytes_on_stack, AllocatorWithStackMemory<Allocator<false>, bytes_on_stack>>;
void parsePattern()
{
PatternActions actions{
{ PatternActionType::KleeneStar }
};
actions.clear();
actions.emplace_back(PatternActionType::KleeneStar);
ParserString special_open_p("(?");
ParserString special_close_p(")");
@ -354,8 +354,6 @@ private:
else
throw_exception("Could not parse pattern, unexpected starting symbol");
}
this->actions = std::move(actions);
}
protected:

1
dbms/include/DB/AggregateFunctions/AggregateFunctionUniq.h
View File

@ -7,7 +7,6 @@
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteBufferFromString.h>
#include <DB/DataTypes/DataTypesNumberFixed.h>
#include <DB/DataTypes/DataTypeString.h>

97
dbms/include/DB/AggregateFunctions/AggregateFunctionsMinMaxAny.h
View File

@ -133,6 +133,16 @@ struct SingleValueDataFixed
else
return false;
}
bool isEqualTo(const Self & to) const
{
return has() && to.value == value;
}
bool isEqualTo(const IColumn & column, size_t row_num) const
{
return has() && static_cast<const ColumnVector<T> &>(column).getData()[row_num] == value;
}
};
@ -334,6 +344,16 @@ struct __attribute__((__packed__, __aligned__(1))) SingleValueDataString
else
return false;
}
bool isEqualTo(const Self & to) const
{
return has() && to.getStringRef() == getStringRef();
}
bool isEqualTo(const IColumn & column, size_t row_num) const
{
return has() && static_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) == getStringRef();
}
};
static_assert(
@ -476,6 +496,16 @@ struct SingleValueDataGeneric
else
return false;
}
bool isEqualTo(const IColumn & column, size_t row_num) const
{
return has() && value == column[row_num];
}
bool isEqualTo(const Self & to) const
{
return has() && to.value == value;
}
};
@ -529,6 +559,73 @@ struct AggregateFunctionAnyLastData : Data
};
/** Implement 'heavy hitters' algorithm.
* Selects most frequent value if its frequency is more than 50% in each thread of execution.
* Otherwise, selects some arbitary value.
* http://www.cs.umd.edu/~samir/498/karp.pdf
*/
template <typename Data>
struct AggregateFunctionAnyHeavyData : Data
{
size_t counter = 0;
using Self = AggregateFunctionAnyHeavyData<Data>;
bool changeIfBetter(const IColumn & column, size_t row_num)
{
if (this->isEqualTo(column, row_num))
{
++counter;
}
else
{
if (counter == 0)
{
this->change(column, row_num);
++counter;
return true;
}
else
--counter;
}
return false;
}
bool changeIfBetter(const Self & to)
{
if (this->isEqualTo(to))
{
counter += to.counter;
}
else
{
if (counter < to.counter)
{
this->change(to);
return true;
}
else
counter -= to.counter;
}
return false;
}
void write(WriteBuffer & buf, const IDataType & data_type) const
{
Data::write(buf, data_type);
writeBinary(counter, buf);
}
void read(ReadBuffer & buf, const IDataType & data_type)
{
Data::read(buf, data_type);
readBinary(counter, buf);
}
static const char * name() { return "anyHeavy"; }
};
template <typename Data>
class AggregateFunctionsSingleValue final : public IUnaryAggregateFunction<Data, AggregateFunctionsSingleValue<Data> >
{

16
dbms/include/DB/AggregateFunctions/UniqVariadicHash.h
View File

@ -88,16 +88,14 @@ struct UniqVariadicHash<true, false>
{
static inline UInt128 apply(size_t num_args, const IColumn ** columns, size_t row_num)
{
SipHash hash;
const IColumn ** column = columns;
const IColumn ** columns_end = column + num_args;
SipHash hash;
while (column < columns_end)
{
StringRef value = (*column)->getDataAt(row_num);
hash.update(reinterpret_cast<const char *>(&value.size), sizeof(value.size));
hash.update(value.data, value.size);
(*column)->updateHashWithValue(row_num, hash);
++column;
}
@ -112,18 +110,16 @@ struct UniqVariadicHash<true, true>
{
static inline UInt128 apply(size_t num_args, const IColumn ** columns, size_t row_num)
{
SipHash hash;
const Columns & tuple_columns = static_cast<const ColumnTuple *>(columns[0])->getColumns();
const ColumnPtr * column = tuple_columns.data();
const ColumnPtr * columns_end = column + num_args;
SipHash hash;
while (column < columns_end)
{
StringRef value = column->get()->getDataAt(row_num);
hash.update(reinterpret_cast<const char *>(&value.size), sizeof(value.size));
hash.update(value.data, value.size);
(*column)->updateHashWithValue(row_num, hash);
++column;
}

66
dbms/include/DB/Client/Connection.h
View File

@ -35,12 +35,12 @@ using ConnectionPtr = std::shared_ptr<Connection>;
using Connections = std::vector<ConnectionPtr>;
/** Соединение с сервером БД для использования в клиенте.
* Как использовать - см. Core/Protocol.h
* (Реализацию на стороне сервера - см. Server/TCPHandler.h)
/** Connection with database server, to use by client.
* How to use - see Core/Protocol.h
* (Implementation of server end - see Server/TCPHandler.h)
*
* В качестве default_database может быть указана пустая строка
* - в этом случае сервер использует свою БД по-умолчанию.
* As 'default_database' empty string could be passed
* - in that case, server will use it's own default database.
*/
class Connection : private boost::noncopyable
{
@ -65,7 +65,7 @@ public:
ping_timeout(ping_timeout_),
log_wrapper(*this)
{
/// Соединеняемся не сразу, а при первой необходимости.
/// Don't connect immediately, only on first need.
if (user.empty())
user = "default";
@ -93,7 +93,7 @@ public:
ping_timeout(ping_timeout_),
log_wrapper(*this)
{
/// Соединеняемся не сразу, а при первой необходимости.
/// Don't connect immediately, only on first need.
if (user.empty())
user = "default";
@ -103,14 +103,14 @@ public:
virtual ~Connection() {};
/// Установить ограничитель сетевого трафика. Один ограничитель может использоваться одновременно для нескольких разных соединений.
/// Set throttler of network traffic. One throttler could be used for multiple connections to limit total traffic.
void setThrottler(const ThrottlerPtr & throttler_)
{
throttler = throttler_;
}
/// Пакет, который может быть получен от сервера.
/// Packet that could be received from server.
struct Packet
{
UInt64 type;
@ -123,46 +123,46 @@ public:
Packet() : type(Protocol::Server::Hello) {}
};
/// Изменить базу данных по умолчанию. Изменения начинают использоваться только при следующем переподключении.
/// Change default database. Changes will take effect on next reconnect.
void setDefaultDatabase(const String & database);
void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
/// Для сообщений в логе и в эксепшенах.
/// For log and exception messages.
const String & getDescription() const;
const String & getHost() const;
UInt16 getPort() const;
const String & getDefaultDatabase() const;
/// Если последний флаг true, то затем необходимо вызвать sendExternalTablesData
/// If last flag is true, you need to call sendExternalTablesData after.
void sendQuery(const String & query, const String & query_id_ = "", UInt64 stage = QueryProcessingStage::Complete,
const Settings * settings = nullptr, bool with_pending_data = false);
void sendCancel();
/// Отправить блок данных, на сервере сохранить во временную таблицу name
/// Send block of data; if name is specified, server will write it to external (temporary) table of that name.
void sendData(const Block & block, const String & name = "");
/// Отправить все содержимое внешних таблиц
/// Send all contents of external (temporary) tables.
void sendExternalTablesData(ExternalTablesData & data);
/// Отправить блок данных, который уже был заранее сериализован (и, если надо, сжат), который следует прочитать из input-а.
/// можно передать размер сериализованного/сжатого блока.
void sendPreparedData(ReadBuffer & input, size_t size, const String & name = "");
/// Send prepared block of data (serialized and, if need, compressed), that will be read from 'input'.
/// You could pass size of serialized/compressed block.
void sendPreparedData(ReadBuffer & input, size_t size, const String & name = "");
/// Проверить, есть ли данные, которые можно прочитать.
/// Check, if has data to read.
bool poll(size_t timeout_microseconds = 0);
/// Проверить, есть ли данные в буфере для чтения.
/// Check, if has data in read buffer.
bool hasReadBufferPendingData() const;
/// Получить пакет от сервера.
/// Receive packet from server.
Packet receivePacket();
/// Если ещё не соединено, или соединение разорвано - соединиться. Если не получилось - кинуть исключение.
/// If not connected yet, or if connection is broken - then connect. If cannot connect - throw an exception.
void forceConnected();
/** Разорвать соединение.
* Это может быть нужно, например, чтобы соединение не осталось висеть в
* рассинхронизированном состоянии (когда кто-то чего-то продолжает ждать) после эксепшена.
/** Disconnect.
* This may be used, if connection is left in unsynchronised state
* (when someone continues to wait for something) after an exception.
*/
void disconnect();
@ -181,8 +181,8 @@ private:
String user;
String password;
/** Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования.
* Иначе адрес резолвится в конструкторе. То есть, DNS балансировка не поддерживается.
/** Address could be resolved beforehand and passed to constructor. Then 'host' and 'port' fields are used just for logging.
* Otherwise address is resolved in constructor. Thus, DNS based load balancing is not supported.
*/
Poco::Net::SocketAddress resolved_address;
@ -204,12 +204,12 @@ private:
std::shared_ptr<WriteBuffer> out;
String query_id;
UInt64 compression; /// Сжимать ли данные при взаимодействии с сервером.
/// каким алгоритмом сжимать данные при INSERT и данные внешних таблиц
UInt64 compression; /// Enable data compression for communication.
/// What compression algorithm to use while sending data for INSERT queries and external tables.
CompressionMethod network_compression_method = CompressionMethod::LZ4;
/** Если не nullptr, то используется, чтобы ограничить сетевой трафик.
* Учитывается только трафик при передаче блоков. Другие пакеты не учитываются.
/** If not nullptr, used to limit network traffic.
* Only traffic for transferring blocks is accounted. Other packets don't.
*/
ThrottlerPtr throttler;
@ -218,15 +218,15 @@ private:
Poco::Timespan send_timeout;
Poco::Timespan ping_timeout;
/// Откуда читать результат выполнения запроса.
/// From where to read query execution result.
std::shared_ptr<ReadBuffer> maybe_compressed_in;
BlockInputStreamPtr block_in;
/// Куда писать данные INSERT-а.
/// Where to write data for INSERT.
std::shared_ptr<WriteBuffer> maybe_compressed_out;
BlockOutputStreamPtr block_out;
/// логгер, создаваемый лениво, чтобы не обращаться к DNS в конструкторе
/// Logger is created lazily, for avoid to run DNS request in constructor.
class LoggerWrapper
{
public:

27
dbms/include/DB/Columns/ColumnAggregateFunction.h
View File

@ -61,7 +61,7 @@ private:
/// Источник. Используется (удерживает источник от уничтожения),
/// если данный столбец создан из другого и использует все или часть его значений.
const std::shared_ptr<const ColumnAggregateFunction> src;
std::shared_ptr<const ColumnAggregateFunction> src;
/// Массив указателей на состояния агрегатных функций, расположенных в пулах.
Container_t data;
@ -201,6 +201,11 @@ public:
throw Exception("Method deserializeAndInsertFromArena is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
throw Exception("Method updateHashWithValue is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
size_t byteSize() const override
{
size_t res = getData().size() * sizeof(getData()[0]);
@ -211,21 +216,31 @@ public:
return res;
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
void insertRangeFrom(const IColumn & from, size_t start, size_t length) override
{
const ColumnAggregateFunction & src_concrete = static_cast<const ColumnAggregateFunction &>(src);
const ColumnAggregateFunction & from_concrete = static_cast<const ColumnAggregateFunction &>(from);
if (start + length > src_concrete.getData().size())
if (start + length > from_concrete.getData().size())
throw Exception("Parameters start = "
+ toString(start) + ", length = "
+ toString(length) + " are out of bound in ColumnAggregateFunction::insertRangeFrom method"
" (data.size() = " + toString(src_concrete.getData().size()) + ").",
" (data.size() = " + toString(from_concrete.getData().size()) + ").",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
if (src && src.get() != &from_concrete)
{
throw Exception("ColumnAggregateFunction could have only one source that owns aggregation states", ErrorCodes::BAD_ARGUMENTS);
}
else
{
/// Keep shared ownership of aggregation states.
src = from_concrete.shared_from_this();
}
auto & data = getData();
size_t old_size = data.size();
data.resize(old_size + length);
memcpy(&data[old_size], &src_concrete.getData()[start], length * sizeof(data[0]));
memcpy(&data[old_size], &from_concrete.getData()[start], length * sizeof(data[0]));
}
void popBack(size_t n) override

14
dbms/include/DB/Columns/ColumnArray.h
View File

@ -4,6 +4,7 @@
#include <DB/Common/Exception.h>
#include <DB/Common/Arena.h>
#include <DB/Common/SipHash.h>
#include <DB/Columns/IColumn.h>
#include <DB/Columns/ColumnsNumber.h>
@ -149,6 +150,16 @@ public:
return pos;
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
size_t array_size = sizeAt(n);
size_t offset = offsetAt(n);
hash.update(reinterpret_cast<const char *>(&array_size), sizeof(array_size));
for (size_t i = 0; i < array_size; ++i)
getData().updateHashWithValue(offset + i, hash);
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
void insert(const Field & x) override
@ -179,7 +190,8 @@ public:
{
auto & offsets = getOffsets();
size_t nested_n = offsets.back() - offsetAt(offsets.size() - n);
getData().popBack(nested_n);
if (nested_n)
getData().popBack(nested_n);
offsets.resize_assume_reserved(offsets.size() - n);
}

8
dbms/include/DB/Columns/ColumnConst.h
View File

@ -136,6 +136,11 @@ public:
throw Exception("Method deserializeAndInsertFromArena is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
throw Exception("Method updateHashWithValue is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{
if (s != filt.size())
@ -289,6 +294,9 @@ public:
/** Преобразование из константы в полноценный столбец */
ColumnPtr convertToFullColumn() const override;
/** Create ColumnTuple of constant columns as elements. */
ColumnPtr convertToTupleOfConstants() const;
void getExtremes(Field & min, Field & max) const override;
};

6
dbms/include/DB/Columns/ColumnFixedString.h
View File

@ -4,6 +4,7 @@
#include <DB/Common/PODArray.h>
#include <DB/Common/Arena.h>
#include <DB/Common/SipHash.h>
#include <DB/Common/memcpySmall.h>
#include <DB/Columns/IColumn.h>
#include <DB/IO/ReadHelpers.h>
@ -143,6 +144,11 @@ public:
return pos + n;
}
void updateHashWithValue(size_t index, SipHash & hash) const override
{
hash.update(reinterpret_cast<const char *>(&chars[n * index]), n);
}
int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int nan_direction_hint) const override
{
const ColumnFixedString & rhs = static_cast<const ColumnFixedString &>(rhs_);

79
dbms/include/DB/Columns/ColumnString.h
View File

@ -9,6 +9,7 @@
#include <DB/Common/Collator.h>
#include <DB/Common/PODArray.h>
#include <DB/Common/Arena.h>
#include <DB/Common/SipHash.h>
#include <DB/Common/memcpySmall.h>
@ -85,43 +86,71 @@ public:
void insert(const Field & x) override
{
const String & s = DB::get<const String &>(x);
size_t old_size = chars.size();
size_t size_to_append = s.size() + 1;
const size_t old_size = chars.size();
const size_t size_to_append = s.size() + 1;
const size_t new_size = old_size + size_to_append;
chars.resize(old_size + size_to_append);
chars.resize(new_size);
memcpy(&chars[old_size], s.c_str(), size_to_append);
offsets.push_back((offsets.size() == 0 ? 0 : offsets.back()) + size_to_append);
offsets.push_back(new_size);
}
void insertFrom(const IColumn & src_, size_t n) override
{
const ColumnString & src = static_cast<const ColumnString &>(src_);
size_t old_size = chars.size();
size_t size_to_append = src.sizeAt(n);
size_t offset = src.offsetAt(n);
chars.resize(old_size + size_to_append);
memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[offset], size_to_append);
offsets.push_back((offsets.size() == 0 ? 0 : offsets.back()) + size_to_append);
if (n != 0)
{
const size_t size_to_append = src.offsets[n] - src.offsets[n - 1];
if (size_to_append == 1)
{
/// shortcut for empty string
chars.push_back(0);
offsets.push_back(chars.size());
}
else
{
const size_t old_size = chars.size();
const size_t offset = src.offsets[n - 1];
const size_t new_size = old_size + size_to_append;
chars.resize(new_size);
memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[offset], size_to_append);
offsets.push_back(new_size);
}
}
else
{
const size_t old_size = chars.size();
const size_t size_to_append = src.offsets[0];
const size_t new_size = old_size + size_to_append;
chars.resize(new_size);
memcpySmallAllowReadWriteOverflow15(&chars[old_size], &src.chars[0], size_to_append);
offsets.push_back(new_size);
}
}
void insertData(const char * pos, size_t length) override
{
size_t old_size = chars.size();
const size_t old_size = chars.size();
const size_t new_size = old_size + length + 1;
chars.resize(old_size + length + 1);
chars.resize(new_size);
memcpy(&chars[old_size], pos, length);
chars[old_size + length] = 0;
offsets.push_back((offsets.size() == 0 ? 0 : offsets.back()) + length + 1);
offsets.push_back(new_size);
}
void insertDataWithTerminatingZero(const char * pos, size_t length) override
{
size_t old_size = chars.size();
const size_t old_size = chars.size();
const size_t new_size = old_size + length;
chars.resize(old_size + length);
chars.resize(new_size);
memcpy(&chars[old_size], pos, length);
offsets.push_back((offsets.size() == 0 ? 0 : offsets.back()) + length);
offsets.push_back(new_size);
}
void popBack(size_t n) override
@ -148,17 +177,27 @@ public:
const char * deserializeAndInsertFromArena(const char * pos) override
{
size_t string_size = *reinterpret_cast<const size_t *>(pos);
const size_t string_size = *reinterpret_cast<const size_t *>(pos);
pos += sizeof(string_size);
size_t old_size = chars.size();
chars.resize(old_size + string_size);
const size_t old_size = chars.size();
const size_t new_size = old_size + string_size;
chars.resize(new_size);
memcpy(&chars[old_size], pos, string_size);
offsets.push_back((offsets.size() == 0 ? 0 : offsets.back()) + string_size);
offsets.push_back(new_size);
return pos + string_size;
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
size_t string_size = sizeAt(n);
size_t offset = offsetAt(n);
hash.update(reinterpret_cast<const char *>(&string_size), sizeof(string_size));
hash.update(reinterpret_cast<const char *>(&chars[offset]), string_size);
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
if (length == 0)

12
dbms/include/DB/Columns/ColumnTuple.h
View File

@ -15,7 +15,11 @@ namespace ErrorCodes
}
/** Столбец, который всего лишь группирует вместе несколько других столбцов.
/** Column, that is just group of few another columns.
*
* For constant Tuples, see ColumnConstTuple.
* Mixed constant/non-constant columns is prohibited in tuple
* for implementation simplicity.
*/
class ColumnTuple final : public IColumn
{
@ -124,6 +128,12 @@ public:
return pos;
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
for (auto & column : columns)
column->updateHashWithValue(n, hash);
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
for (size_t i = 0; i < columns.size(); ++i)

6
dbms/include/DB/Columns/ColumnVector.h
View File

@ -5,6 +5,7 @@
#include <DB/Common/Exception.h>
#include <DB/Common/Arena.h>
#include <DB/Common/SipHash.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/WriteHelpers.h>
@ -182,6 +183,11 @@ public:
return pos + sizeof(T);
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
hash.update(reinterpret_cast<const char *>(&data[n]), sizeof(T));
}
size_t byteSize() const override
{
return data.size() * sizeof(data[0]);

13
dbms/include/DB/Columns/IColumn.h
View File

@ -10,6 +10,9 @@
#include <DB/Core/StringRef.h>
class SipHash;
namespace DB
{
@ -162,7 +165,8 @@ public:
/** Удалить одно или несколько значений с конца.
* Используется, чтобы сделать некоторые операции exception-safe,
* когда после вставки значения сделать что-то ещё не удалось, и нужно откатить вставку.
* Если столбец пуст - поведение не определено.
* Если столбец имеет меньше n значений - поведение не определено.
* Если n == 0 - поведение не определено.
*/
virtual void popBack(size_t n) = 0;
@ -180,10 +184,11 @@ public:
*/
virtual const char * deserializeAndInsertFromArena(const char * pos) = 0;
/** Соединить столбец с одним или несколькими другими.
* Используется при склейке маленьких блоков.
/** Update state of hash function with value at index n.
* On subsequent calls of this method for sequence of column values of arbitary types,
* passed bytes to hash must identify sequence of values unambiguously.
*/
//virtual void merge(const Columns & columns) = 0;
virtual void updateHashWithValue(size_t n, SipHash & hash) const = 0;
/** Оставить только значения, соответствующие фильтру.
* Используется для операции WHERE / HAVING.

5
dbms/include/DB/Columns/IColumnDummy.h
View File

@ -48,6 +48,11 @@ public:
throw Exception("Method deserializeAndInsertFromArena is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void updateHashWithValue(size_t n, SipHash & hash) const override
{
throw Exception("Method updateHashWithValue is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void getExtremes(Field & min, Field & max) const override
{
throw Exception("Method getExtremes is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);

12
dbms/include/DB/Common/AIO.h
View File

@ -78,14 +78,14 @@ class AIOContextPool : public Singleton<AIOContextPool>
AIOContext aio_context{max_concurrent_events};
using id_t = size_t;
using bytes_read_t = ssize_t;
using ID = size_t;
using BytesRead = ssize_t;
/// Autoincremental id used to identify completed requests
id_t id{};
ID id{};
mutable std::mutex mutex;
mutable std::condition_variable have_resources;
std::map<id_t, std::promise<bytes_read_t>> promises;
std::map<ID, std::promise<BytesRead>> promises;
std::atomic<bool> cancelled{false};
std::thread io_completion_monitor{&AIOContextPool::doMonitor, this};
@ -188,7 +188,7 @@ class AIOContextPool : public Singleton<AIOContextPool>
public:
/// Request AIO read operation for iocb, returns a future with number of bytes read
std::future<bytes_read_t> post(struct iocb & iocb)
std::future<BytesRead> post(struct iocb & iocb)
{
std::unique_lock<std::mutex> lock{mutex};
@ -196,7 +196,7 @@ public:
const auto request_id = id++;
/// create a promise and put request in "queue"
promises.emplace(request_id, std::promise<bytes_read_t>{});
promises.emplace(request_id, std::promise<BytesRead>{});
/// store id in AIO request for further identification
iocb.aio_data = request_id;

12
dbms/include/DB/Common/Allocator.h
View File

@ -164,6 +164,12 @@ public:
return buf;
}
protected:
static constexpr size_t getStackThreshold()
{
return 0;
}
};
@ -209,6 +215,12 @@ public:
memcpy(new_buf, buf, old_size);
return new_buf;
}
protected:
static constexpr size_t getStackThreshold()
{
return N;
}
};

2
dbms/include/DB/Common/ArenaWithFreeLists.h
View File

@ -34,7 +34,7 @@ private:
/// Получить индекс в массиве freelist-ов для заданного размера.
static size_t findFreeListIndex(const size_t size)
{
return size <= 8 ? 2 : bit_scan_reverse(size - 1);
return size <= 8 ? 2 : bitScanReverse(size - 1);
}
/// Для выделения блоков не слишком большого размера используется Arena.

28
dbms/include/DB/Common/BitHelpers.h
View File

@ -1,6 +1,32 @@
#pragma once
inline unsigned int bit_scan_reverse(unsigned int x)
#include <cstdint>
/** Returns log2 of number, rounded down.
* Compiles to single 'bsr' instruction on x86.
* For zero argument, result is unspecified.
*/
inline unsigned int bitScanReverse(unsigned int x)
{
return sizeof(unsigned int) * 8 - 1 - __builtin_clz(x);
}
/** For zero argument, result is zero.
* For arguments with most significand bit set, result is zero.
* For other arguments, returns value, rounded up to power of two.
*/
inline size_t roundUpToPowerOfTwoOrZero(size_t n)
{
--n;
n |= n >> 1;
n |= n >> 2;
n |= n >> 4;
n |= n >> 8;
n |= n >> 16;
n |= n >> 32;
++n;
return n;
}

17
dbms/include/DB/Common/CurrentMetrics.h
View File

@ -2,6 +2,7 @@
#include <cstdint>
#include <utility>
#include <atomic>
/** Позволяет считать количество одновременно происходящих событий или текущее значение какой-либо метрики.
@ -65,7 +66,7 @@ namespace CurrentMetrics
using Value = int64_t;
/// Счётчики - текущие значения метрик.
extern Value values[END];
extern std::atomic<Value> values[END];
/// Выставить значение указанной метрики.
@ -77,7 +78,7 @@ namespace CurrentMetrics
/// Прибавить величину к значению указанной метрики. Вы затем должны вычесть величину самостоятельно. Или см. ниже class Increment.
inline void add(Metric metric, Value value = 1)
{
__sync_fetch_and_add(&values[metric], value);
values[metric] += value;
}
inline void sub(Metric metric, Value value = 1)
@ -89,13 +90,13 @@ namespace CurrentMetrics
class Increment
{
private:
Value * what;
std::atomic<Value> * what;
Value amount;
Increment(Value * what, Value amount)
Increment(std::atomic<Value> * what, Value amount)
: what(what), amount(amount)
{
__sync_fetch_and_add(what, amount);
*what += amount;
}
public:
@ -105,7 +106,7 @@ namespace CurrentMetrics
~Increment()
{
if (what)
__sync_fetch_and_sub(what, amount);
*what -= amount;
}
Increment(Increment && old)
@ -123,14 +124,14 @@ namespace CurrentMetrics
void changeTo(Value new_amount)
{
__sync_fetch_and_add(what, new_amount - amount);
*what += new_amount - amount;
amount = new_amount;
}
/// Уменьшить значение раньше вызова деструктора.
void destroy()
{
__sync_fetch_and_sub(what, amount);
*what -= amount;
what = nullptr;
}
};

2
dbms/include/DB/Common/ExternalTable.h
View File

@ -50,7 +50,7 @@ public:
column.name = structure[i].first;
column.type = data_type_factory.get(structure[i].second);
column.column = column.type->createColumn();
sample_block.insert(column);
sample_block.insert(std::move(column));
}
}

10
dbms/include/DB/Common/HTMLForm.h
View File

@ -6,19 +6,19 @@
#include <Poco/URI.h>
/** Почему-то при методе POST, Poco::Net::HTMLForm не считывает параметры из URL, а считывает только из тела.
* Этот помошник позволяет считывать параметры только из URL.
*/
/** Somehow, in case of POST, Poco::Net::HTMLForm doesn't read parameters from URL, only from body.
* This helper allows to read parameters just from URL.
*/
struct HTMLForm : public Poco::Net::HTMLForm
{
HTMLForm(Poco::Net::HTTPRequest & request)
HTMLForm(const Poco::Net::HTTPRequest & request)
{
Poco::URI uri(request.getURI());
std::istringstream istr(uri.getRawQuery());
readUrl(istr);
}
HTMLForm(Poco::URI & uri)
HTMLForm(const Poco::URI & uri)
{
std::istringstream istr(uri.getRawQuery());
readUrl(istr);

4
dbms/include/DB/Common/Increment.h
View File

@ -71,7 +71,7 @@ private:
*/
struct SimpleIncrement : private boost::noncopyable
{
UInt64 value;
std::atomic<UInt64> value;
SimpleIncrement(UInt64 start = 0) : value(start) {}
@ -82,6 +82,6 @@ struct SimpleIncrement : private boost::noncopyable
UInt64 get()
{
return __sync_add_and_fetch(&value, 1);
return ++value;
}
};

11
dbms/include/DB/Common/MemoryTracker.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <atomic>
#include <common/Common.h>
@ -9,9 +10,9 @@
*/
class MemoryTracker
{
Int64 amount = 0;
Int64 peak = 0;
Int64 limit = 0;
std::atomic<Int64> amount {0};
std::atomic<Int64> peak {0};
Int64 limit {0};
/// В целях тестирования exception safety - кидать исключение при каждом выделении памяти с указанной вероятностью.
double fault_probability = 0;
@ -43,12 +44,12 @@ public:
Int64 get() const
{
return amount;
return amount.load(std::memory_order_relaxed);
}
Int64 getPeak() const
{
return peak;
return peak.load(std::memory_order_relaxed);
}
void setLimit(Int64 limit_)

172
dbms/include/DB/Common/PODArray.h
View File

@ -13,6 +13,7 @@
#include <DB/Common/Allocator.h>
#include <DB/Common/Exception.h>
#include <DB/Common/BitHelpers.h>
namespace DB
@ -60,23 +61,9 @@ private:
/// Минимальное количество памяти, которое нужно выделить для num_elements элементов, включая padding.
static size_t minimum_memory_for_elements(size_t num_elements) { return byte_size(num_elements) + pad_right; }
static size_t round_up_to_power_of_two(size_t n)
{
--n;
n |= n >> 1;
n |= n >> 2;
n |= n >> 4;
n |= n >> 8;
n |= n >> 16;
n |= n >> 32;
++n;
return n;
}
void alloc_for_num_elements(size_t num_elements)
{
alloc(round_up_to_power_of_two(minimum_memory_for_elements(num_elements)));
alloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(num_elements)));
}
void alloc(size_t bytes)
@ -109,6 +96,17 @@ private:
c_end_of_storage = c_start + bytes - pad_right;
}
bool isInitialized() const
{
return (c_start != nullptr) && (c_end != nullptr) && (c_end_of_storage != nullptr);
}
bool isAllocatedFromStack() const
{
constexpr size_t stack_threshold = TAllocator::getStackThreshold();
return (stack_threshold > 0) && (allocated_size() <= stack_threshold);
}
public:
using value_type = T;
@ -130,55 +128,46 @@ public:
PODArray() {}
PODArray(size_t n)
PODArray(size_t n)
{
alloc_for_num_elements(n);
c_end += byte_size(n);
}
PODArray(size_t n, const T & x)
PODArray(size_t n, const T & x)
{
alloc_for_num_elements(n);
assign(n, x);
}
PODArray(const_iterator from_begin, const_iterator from_end)
PODArray(const_iterator from_begin, const_iterator from_end)
{
alloc_for_num_elements(from_end - from_begin);
insert(from_begin, from_end);
}
~PODArray()
~PODArray()
{
dealloc();
}
PODArray(PODArray && other)
{
c_start = other.c_start;
c_end = other.c_end;
c_end_of_storage = other.c_end_of_storage;
other.c_start = nullptr;
other.c_end = nullptr;
other.c_end_of_storage = nullptr;
this->swap(other);
}
PODArray & operator=(PODArray && other)
{
std::swap(c_start, other.c_start);
std::swap(c_end, other.c_end);
std::swap(c_end_of_storage, other.c_end_of_storage);
this->swap(other);
return *this;
}
T * data() { return t_start(); }
const T * data() const { return t_start(); }
size_t size() const { return t_end() - t_start(); }
bool empty() const { return t_end() == t_start(); }
size_t capacity() const { return t_end_of_storage() - t_start(); }
size_t size() const { return t_end() - t_start(); }
bool empty() const { return t_end() == t_start(); }
size_t capacity() const { return t_end_of_storage() - t_start(); }
T & operator[] (size_t n) { return t_start()[n]; }
const T & operator[] (size_t n) const { return t_start()[n]; }
@ -198,7 +187,7 @@ public:
void reserve(size_t n)
{
if (n > capacity())
realloc(round_up_to_power_of_two(minimum_memory_for_elements(n)));
realloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(n)));
}
void reserve()
@ -278,7 +267,7 @@ public:
{
size_t required_capacity = size() + (from_end - from_begin);
if (required_capacity > capacity())
reserve(round_up_to_power_of_two(required_capacity));
reserve(roundUpToPowerOfTwoOrZero(required_capacity));
insert_assume_reserved(from_begin, from_end);
}
@ -288,7 +277,7 @@ public:
{
size_t required_capacity = size() + (from_end - from_begin);
if (required_capacity > capacity())
reserve(round_up_to_power_of_two(required_capacity));
reserve(roundUpToPowerOfTwoOrZero(required_capacity));
size_t bytes_to_copy = byte_size(from_end - from_begin);
size_t bytes_to_move = (end() - it) * sizeof(T);
@ -310,9 +299,107 @@ public:
void swap(PODArray & rhs)
{
std::swap(c_start, rhs.c_start);
std::swap(c_end, rhs.c_end);
std::swap(c_end_of_storage, rhs.c_end_of_storage);
/// Swap two PODArray objects, arr1 and arr2, that satisfy the following conditions:
/// - The elements of arr1 are stored on stack.
/// - The elements of arr2 are stored on heap.
auto swap_stack_heap = [](PODArray & arr1, PODArray & arr2)
{
size_t stack_size = arr1.size();
size_t stack_allocated = arr1.allocated_size();
size_t heap_size = arr2.size();
size_t heap_allocated = arr2.allocated_size();
/// Keep track of the stack content we have to copy.
char * stack_c_start = arr1.c_start;
/// arr1 takes ownership of the heap memory of arr2.
arr1.c_start = arr2.c_start;
arr1.c_end_of_storage = arr1.c_start + heap_allocated - arr1.pad_right;
arr1.c_end = arr1.c_start + byte_size(heap_size);
/// Allocate stack space for arr2.
arr2.alloc(stack_allocated);
/// Copy the stack content.
memcpy(arr2.c_start, stack_c_start, byte_size(stack_size));
arr2.c_end = arr2.c_start + byte_size(stack_size);
};
auto do_move = [](PODArray & src, PODArray & dest)
{
if (src.isAllocatedFromStack())
{
dest.dealloc();
dest.alloc(src.allocated_size());
memcpy(dest.c_start, src.c_start, byte_size(src.size()));
dest.c_end = dest.c_start + (src.c_end - src.c_start);
src.c_start = nullptr;
src.c_end = nullptr;
src.c_end_of_storage = nullptr;
}
else
{
std::swap(dest.c_start, src.c_start);
std::swap(dest.c_end, src.c_end);
std::swap(dest.c_end_of_storage, src.c_end_of_storage);
}
};
if (!isInitialized() && !rhs.isInitialized())
return;
else if (!isInitialized() && rhs.isInitialized())
{
do_move(rhs, *this);
return;
}
else if (isInitialized() && !rhs.isInitialized())
{
do_move(*this, rhs);
return;
}
if (isAllocatedFromStack() && rhs.isAllocatedFromStack())
{
size_t min_size = std::min(size(), rhs.size());
size_t max_size = std::max(size(), rhs.size());
for (size_t i = 0; i < min_size; ++i)
std::swap(this->operator[](i), rhs[i]);
if (size() == max_size)
{
for (size_t i = min_size; i < max_size; ++i)
rhs[i] = this->operator[](i);
}
else
{
for (size_t i = min_size; i < max_size; ++i)
this->operator[](i) = rhs[i];
}
size_t lhs_size = size();
size_t lhs_allocated = allocated_size();
size_t rhs_size = rhs.size();
size_t rhs_allocated = rhs.allocated_size();
c_end_of_storage = c_start + rhs_allocated - pad_right;
rhs.c_end_of_storage = rhs.c_start + lhs_allocated - pad_right;
c_end = c_start + byte_size(rhs_size);
rhs.c_end = rhs.c_start + byte_size(lhs_size);
}
else if (isAllocatedFromStack() && !rhs.isAllocatedFromStack())
swap_stack_heap(*this, rhs);
else if (!isAllocatedFromStack() && rhs.isAllocatedFromStack())
swap_stack_heap(rhs, *this);
else
{
std::swap(c_start, rhs.c_start);
std::swap(c_end, rhs.c_end);
std::swap(c_end_of_storage, rhs.c_end_of_storage);
}
}
void assign(size_t n, const T & x)
@ -326,7 +413,7 @@ public:
{
size_t required_capacity = from_end - from_begin;
if (required_capacity > capacity())
reserve(round_up_to_power_of_two(required_capacity));
reserve(roundUpToPowerOfTwoOrZero(required_capacity));
size_t bytes_to_copy = byte_size(required_capacity);
memcpy(c_start, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
@ -365,6 +452,11 @@ public:
}
};
template <typename T, size_t INITIAL_SIZE, typename TAllocator, size_t pad_right_>
void swap(PODArray<T, INITIAL_SIZE, TAllocator, pad_right_> & lhs, PODArray<T, INITIAL_SIZE, TAllocator, pad_right_> & rhs)
{
lhs.swap(rhs);
}
/** Для столбцов. Padding-а хватает, чтобы читать и писать xmm-регистр по адресу последнего элемента. */
template <typename T, size_t INITIAL_SIZE = 4096, typename TAllocator = Allocator<false>>

31
dbms/include/DB/Common/PoolWithFailoverBase.h
View File

@ -5,6 +5,7 @@
#include <DB/Common/ProfileEvents.h>
#include <DB/Common/NetException.h>
#include <DB/Common/Exception.h>
#include <DB/Common/randomSeed.h>
#include <DB/Interpreters/Settings.h>
@ -13,7 +14,6 @@ namespace DB
namespace ErrorCodes
{
extern const int ALL_CONNECTION_TRIES_FAILED;
extern const int CANNOT_CLOCK_GETTIME;
extern const int LOGICAL_ERROR;
}
}
@ -154,11 +154,7 @@ protected:
public:
PoolWithErrorCount(const NestedPoolPtr & pool_) : pool(pool_)
{
struct timespec times;
if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &times))
DB::throwFromErrno("Cannot clock_gettime.", DB::ErrorCodes::CANNOT_CLOCK_GETTIME);
srand48_r(reinterpret_cast<intptr_t>(this) ^ times.tv_nsec, &rand_state);
srand48_r(randomSeed(), &rand_state);
}
void randomize()
@ -173,13 +169,20 @@ protected:
{
static bool compare(const State & lhs, const State & rhs)
{
return std::tie(lhs.priority, lhs.error_count, lhs.random)
< std::tie(rhs.priority, rhs.error_count, rhs.random);
return std::forward_as_tuple(lhs.priority, lhs.error_count.load(std::memory_order_relaxed), lhs.random)
< std::forward_as_tuple(rhs.priority, rhs.error_count.load(std::memory_order_relaxed), rhs.random);
}
Int64 priority = 0;
UInt64 error_count = 0;
UInt32 random = 0;
Int64 priority {0};
std::atomic<UInt64> error_count {0};
UInt32 random {0};
State() {}
State(const State & other)
: priority(other.priority),
error_count(other.error_count.load(std::memory_order_relaxed)),
random(other.random) {}
};
public:
@ -237,7 +240,9 @@ protected:
else if (shift_amount)
{
for (auto & pool : *this)
pool.state.error_count >>= shift_amount;
pool.state.error_count.store(
pool.state.error_count.load(std::memory_order_relaxed) >> shift_amount,
std::memory_order_relaxed);
}
}
}
@ -319,7 +324,7 @@ private:
fail_messages << fail_message.str() << std::endl;
__sync_fetch_and_add(&pool_ptrs[i].pool->state.error_count, 1);
++pool_ptrs[i].pool->state.error_count;
}
}

5
dbms/include/DB/Common/ProfileEvents.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <stddef.h>
#include <atomic>
/** Позволяет считать количество различных событий, произошедших в программе
@ -125,13 +126,13 @@ namespace ProfileEvents
/// Счётчики - сколько раз каждое из событий произошло.
extern size_t counters[END];
extern std::atomic<size_t> counters[END];
/// Увеличить счётчик события. Потокобезопасно.
inline void increment(Event event, size_t amount = 1)
{
__sync_fetch_and_add(&counters[event], amount);
counters[event] += amount;
}
}

86
dbms/include/DB/Common/StringUtils.h Normal file
View File

@ -0,0 +1,86 @@
#pragma once
#include <string>
#include <cstring>
namespace detail
{
bool startsWith(const std::string & s, const char * prefix, size_t prefix_size);
bool endsWith(const std::string & s, const char * suffix, size_t suffix_size);
}
inline bool startsWith(const std::string & s, const std::string & prefix)
{
return detail::startsWith(s, prefix.data(), prefix.size());
}
inline bool endsWith(const std::string & s, const std::string & suffix)
{
return detail::endsWith(s, suffix.data(), suffix.size());
}
/// strlen evaluated compile-time.
inline bool startsWith(const std::string & s, const char * prefix)
{
return detail::startsWith(s, prefix, strlen(prefix));
}
inline bool endsWith(const std::string & s, const char * suffix)
{
return detail::endsWith(s, suffix, strlen(suffix));
}
/// More efficient than libc, because doesn't respect locale.
inline bool isASCII(char c)
{
return static_cast<unsigned char>(c) < 0x80;
}
inline bool isAlphaASCII(char c)
{
return (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z');
}
inline bool isNumericASCII(char c)
{
return (c >= '0' && c <= '9');
}
inline bool isAlphaNumericASCII(char c)
{
return isAlphaASCII(c)
|| isNumericASCII(c);
}
inline bool isWordCharASCII(char c)
{
return isAlphaNumericASCII(c)
|| c == '_';
}
inline bool isWhitespaceASCII(char c)
{
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
/// Works assuming isAlphaASCII.
inline char toLowerIfAlphaASCII(char c)
{
return c | 0x20;
}
inline char toUpperIfAlphaASCII(char c)
{
return c & (~0x20);
}
inline char alternateCaseIfAlphaASCII(char c)
{
return c ^ 0x20;
}

61
dbms/include/DB/Common/ThreadPool.h Normal file
View File

@ -0,0 +1,61 @@
#pragma once
#include <cstdint>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <functional>
#include <queue>
#include <vector>
/** Very simple thread pool similar to boost::threadpool.
* Advantages:
* - catches exceptions and rethrows on wait.
*/
class ThreadPool
{
private:
using Job = std::function<void()>;
public:
/// Size is constant, all threads are created immediately.
ThreadPool(size_t m_size);
/// Add new job. Locks until free thread in pool become available or exception in one of threads was thrown.
/// If an exception in some thread was thrown, method silently returns, and exception will be rethrown only on call to 'wait' function.
void schedule(Job job);
/// Wait for all currently active jobs to be done.
/// You may call schedule and wait many times in arbitary order.
/// If any thread was throw an exception, it will be rethrown from this method.
/// List of exceptions is not cleared: on subsequent calls to wait, same exception will be rethrown.
void wait();
/// Waits for all threads. Doesn't rethrow exceptions (use 'wait' method to rethrow exceptions).
/// You should not destroy object while calling schedule or wait methods from another threads.
~ThreadPool();
size_t size() const { return m_size; }
/// Returns number of active jobs.
size_t active() const;
private:
mutable std::mutex mutex;
std::condition_variable has_free_thread;
std::condition_variable has_new_job_or_shutdown;
const size_t m_size;
size_t active_jobs = 0;
bool shutdown = false;
std::queue<Job> jobs;
std::vector<std::thread> threads;
std::vector<std::exception_ptr> exceptions; /// NOTE Saving many exceptions but we rethrow just first one.
void worker();
};

2
dbms/include/DB/Common/UTF8Helpers.h
View File

@ -42,7 +42,7 @@ inline std::size_t seqLength(const UInt8 first_octet)
return 1;
const std::size_t bits = 8;
const auto first_zero = bit_scan_reverse(static_cast<UInt8>(~first_octet));
const auto first_zero = bitScanReverse(static_cast<UInt8>(~first_octet));
return bits - 1 - first_zero;
}

44
dbms/include/DB/Common/Volnitsky.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <DB/Common/StringSearcher.h>
#include <DB/Common/StringUtils.h>
#include <Poco/UTF8Encoding.h>
#include <Poco/Unicode.h>
#include <ext/range.hpp>
@ -56,35 +57,6 @@ protected:
static constexpr auto min_haystack_size_for_algorithm = 20000;
const bool fallback; /// Нужно ли использовать fallback алгоритм.
/// Эти функции эффективнее, чем соответствующие из libc, так как не учитывают локаль (что нам и надо).
bool isascii(char c)
{
return static_cast<unsigned char>(c) < 0x80;
}
bool isalpha(char c)
{
return (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z');
}
/// Работает корректно только при условии isalpha.
char tolower(char c)
{
return c | 0x20;
}
char toupper(char c)
{
return c & (~0x20);
}
char alternate(char c)
{
return c ^ 0x20;
}
public:
/** haystack_size_hint - ожидаемый суммарный размер haystack при вызовах search. Можно не указывать.
* Если указать его достаточно маленьким, то будет использован fallback алгоритм,
@ -179,32 +151,32 @@ protected:
n = toNGram(pos);
const auto c0_al = isalpha(chars.c0);
const auto c1_al = isalpha(chars.c1);
const auto c0_al = isAlphaASCII(chars.c0);
const auto c1_al = isAlphaASCII(chars.c1);
if (c0_al && c1_al)
{
/// 4 combinations: AB, aB, Ab, ab
putNGramBase(n, offset);
chars.c0 = alternate(chars.c0);
chars.c0 = alternateCaseIfAlphaASCII(chars.c0);
putNGramBase(n, offset);
chars.c1 = alternate(chars.c1);
chars.c1 = alternateCaseIfAlphaASCII(chars.c1);
putNGramBase(n, offset);
chars.c0 = alternate(chars.c0);
chars.c0 = alternateCaseIfAlphaASCII(chars.c0);
putNGramBase(n, offset);
}
else if (c0_al)
{
/// 2 combinations: A1, a1
putNGramBase(n, offset);
chars.c0 = alternate(chars.c0);
chars.c0 = alternateCaseIfAlphaASCII(chars.c0);
putNGramBase(n, offset);
}
else if (c1_al)
{
/// 2 combinations: 0B, 0b
putNGramBase(n, offset);
chars.c1 = alternate(chars.c1);
chars.c1 = alternateCaseIfAlphaASCII(chars.c1);
putNGramBase(n, offset);
}
else

3
dbms/include/DB/Common/escapeForFileName.h
View File

@ -2,6 +2,7 @@
#include <string>
#include <DB/IO/ReadHelpers.h>
#include <DB/Common/StringUtils.h>
namespace DB
@ -19,7 +20,7 @@ inline std::string escapeForFileName(const std::string & s)
{
char c = *pos;
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c == '_'))
if (isWordCharASCII(c))
res += c;
else
{

2
dbms/include/DB/Common/getNumberOfPhysicalCPUCores.h
View File

@ -1,4 +1,4 @@
#pragma once
/// Получить количество ядер CPU без учёта hyper-threading.
/// Get number of CPU cores without hyper-threading.
unsigned getNumberOfPhysicalCPUCores();

6
dbms/include/DB/Common/randomSeed.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include <cstdint>
/** Returns a number suitable as seed for PRNG. Use clock_gettime, pid and so on. */
uint64_t randomSeed();

13
dbms/include/DB/Common/unaligned.h Normal file
View File

@ -0,0 +1,13 @@
#pragma once
#include <DB/Core/Defines.h>
#include <string.h>
template <typename T>
inline T unalignedLoad(const void * address)
{
T res {};
memcpy(&res, address, sizeof(res));
return res;
}

57
dbms/include/DB/Core/Block.h
View File

@ -2,14 +2,13 @@
#include <vector>
#include <map>
#include <list>
#include <initializer_list>
#include <DB/Core/BlockInfo.h>
#include <DB/Core/ColumnWithTypeAndName.h>
#include <DB/Core/NamesAndTypes.h>
#include <DB/Common/Exception.h>
#include <DB/Core/ColumnWithTypeAndName.h>
#include <DB/Core/ColumnsWithTypeAndName.h>
#include <DB/Common/Exception.h>
namespace DB
@ -24,15 +23,12 @@ class Context;
class Block
{
public:
using Container_t = std::list<ColumnWithTypeAndName>;
using IndexByPosition_t = std::vector<Container_t::iterator>;
using IndexByName_t = std::map<String, Container_t::iterator>;
private:
Container_t data;
IndexByPosition_t index_by_position;
IndexByName_t index_by_name;
using Container = std::vector<ColumnWithTypeAndName>;
using IndexByName = std::map<String, size_t>;
Container data;
IndexByName index_by_name;
public:
BlockInfo info;
@ -40,26 +36,23 @@ public:
Block() = default;
Block(std::initializer_list<ColumnWithTypeAndName> il) : data{il}
{
index_by_position.reserve(il.size());
for (auto it = std::begin(data); it != std::end(data); ++it)
size_t i = 0;
for (const auto & elem : il)
{
index_by_name[it->name] = it;
index_by_position.push_back(it);
index_by_name[elem.name] = i;
++i;
}
}
/// нужны, чтобы правильно скопировались индексы
Block(const Block & other);
Block(Block && other) = default;
Block & operator= (const Block & other);
Block & operator= (Block && other) = default;
/// вставить столбец в заданную позицию
void insert(size_t position, const ColumnWithTypeAndName & elem);
void insert(size_t position, ColumnWithTypeAndName && elem);
/// вставить столбец в конец
void insert(const ColumnWithTypeAndName & elem);
void insert(ColumnWithTypeAndName && elem);
/// вставить столбец в конец, если столбца с таким именем ещё нет
void insertUnique(const ColumnWithTypeAndName & elem);
void insertUnique(ColumnWithTypeAndName && elem);
/// удалить столбец в заданной позиции
void erase(size_t position);
/// удалить столбец с заданным именем
@ -67,11 +60,13 @@ public:
/// Добавляет в блок недостающие столбцы со значениями по-умолчанию
void addDefaults(const NamesAndTypesList & required_columns);
/// References are invalidated after calling functions above.
ColumnWithTypeAndName & getByPosition(size_t position);
const ColumnWithTypeAndName & getByPosition(size_t position) const;
ColumnWithTypeAndName & unsafeGetByPosition(size_t position) { return *index_by_position[position]; }
const ColumnWithTypeAndName & unsafeGetByPosition(size_t position) const { return *index_by_position[position]; }
ColumnWithTypeAndName & unsafeGetByPosition(size_t position) { return data[position]; }
const ColumnWithTypeAndName & unsafeGetByPosition(size_t position) const { return data[position]; }
ColumnWithTypeAndName & getByName(const std::string & name);
const ColumnWithTypeAndName & getByName(const std::string & name) const;
@ -92,13 +87,13 @@ public:
*/
size_t rowsInFirstColumn() const;
size_t columns() const { return index_by_position.size(); }
size_t columns() const { return data.size(); }
/// Приблизительное количество байт в оперативке - для профайлинга.
size_t bytes() const;
operator bool() const { return !index_by_position.empty(); }
bool operator!() const { return index_by_position.empty(); }
operator bool() const { return !data.empty(); }
bool operator!() const { return data.empty(); }
/** Получить список имён столбцов через запятую. */
std::string dumpNames() const;
@ -121,6 +116,16 @@ public:
void clear();
void swap(Block & other) noexcept;
/** Some column implementations (ColumnArray) may have shared parts between different columns
* (common array sizes of elements of nested data structures).
* Before doing mutating operations on such columns, you must unshare that parts.
* Also unsharing columns, if whole columns are shared_ptrs pointing to same instances.
*/
void unshareColumns();
private:
void eraseImpl(size_t position);
};
using Blocks = std::vector<Block>;

32
dbms/include/DB/Core/Defines.h
View File

@ -6,7 +6,7 @@
#define DBMS_DEFAULT_HOST "localhost"
#define DBMS_DEFAULT_PORT 9000
#define DBMS_DEFAULT_PORT_STR "9000"
#define DBMS_DEFAULT_PORT_STR #DBMS_DEFAULT_PORT
#define DBMS_DEFAULT_HTTP_PORT 8123
#define DBMS_DEFAULT_CONNECT_TIMEOUT_SEC 10
#define DBMS_DEFAULT_CONNECT_TIMEOUT_WITH_FAILOVER_MS 50
@ -53,9 +53,9 @@
#define DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES 3
/// каждый период уменьшаем счетчик ошибок в 2 раза
/// слишком маленький период может приводить, что ошибки исчезают сразу после создания.
#define DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD (2*DBMS_DEFAULT_SEND_TIMEOUT_SEC)
#define DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD (2 * DBMS_DEFAULT_SEND_TIMEOUT_SEC)
#define DEFAULT_QUERIES_QUEUE_WAIT_TIME_MS 5000 /// Максимальное время ожидания в очереди запросов.
#define DBMS_DEFAULT_BACKGROUND_POOL_SIZE 6
#define DBMS_DEFAULT_BACKGROUND_POOL_SIZE 6
/// Используется в методе reserve, когда известно число строк, но неизвестны их размеры.
#define DBMS_APPROX_STRING_SIZE 64
@ -63,16 +63,11 @@
/// Суффикс имени для столбца, содержащего смещения массива.
#define ARRAY_SIZES_COLUMN_NAME_SUFFIX ".size"
#define DBMS_MIN_REVISION_WITH_PER_QUERY_SETTINGS 28558
#define DBMS_MIN_REVISION_WITH_PROFILING_PACKET 32029
#define DBMS_MIN_REVISION_WITH_HEADER_BLOCK 32881
#define DBMS_MIN_REVISION_WITH_USER_PASSWORD 34482
#define DBMS_MIN_REVISION_WITH_TOTALS_EXTREMES 35265
#define DBMS_MIN_REVISION_WITH_STRING_QUERY_ID 39002
#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50264
#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS 51554
#define DBMS_MIN_REVISION_WITH_BLOCK_INFO 51903
/// Версия TCP протокола ClickHouse
/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
#define DBMS_TCP_PROTOCOL_VERSION 53694
#define DBMS_DISTRIBUTED_DIRECTORY_MONITOR_SLEEP_TIME_MS 100
@ -90,3 +85,20 @@
#if !defined(__x86_64__) && !defined(__aarch64__)
#error PLATFORM_NOT_SUPPORTED
#endif
/// Check for presence of address sanitizer
#if defined(__has_feature)
#if __has_feature(address_sanitizer)
#define ADDRESS_SANITIZER 1
#endif
#elif defined(__SANITIZE_ADDRESS__)
#define ADDRESS_SANITIZER 1
#endif
#if defined(__has_feature)
#if __has_feature(thread_sanitizer)
#define THREAD_SANITIZER 1
#endif
#elif defined(__SANITIZE_THREAD__)
#define THREAD_SANITIZER 1
#endif

42
dbms/include/DB/Core/FieldVisitors.h
View File

@ -254,16 +254,16 @@ public:
UInt64 stringToDateOrDateTime(const String & s);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
/** Более точное сравнение.
* Отличается от Field::operator< и Field::operator== тем, что сравнивает значения разных числовых типов между собой.
* Правила сравнения - такие же, что и в FunctionsComparison.
* В том числе, сравнение знаковых и беззнаковых оставляем UB.
/** More precise comparison, used for index.
* Differs from Field::operator< and Field::operator== in that it also compares values of different types.
* Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query).
* Except in cases when comparing signed and unsigned integers, which is unspecified behavior in FunctionsComparison,
* and when comparing integers and floats. Comparison is accurate here.
*/
class FieldVisitorAccurateEquals : public StaticVisitor<bool>
{
using Double128 = long double; /// Non portable. Must have 64 bit mantissa to provide accurate comparisons.
public:
bool operator() (const Null & l, const Null & r) const { return true; }
bool operator() (const Null & l, const UInt64 & r) const { return false; }
@ -275,23 +275,23 @@ public:
bool operator() (const UInt64 & l, const Null & r) const { return false; }
bool operator() (const UInt64 & l, const UInt64 & r) const { return l == r; }
bool operator() (const UInt64 & l, const Int64 & r) const { return l == r; }
bool operator() (const UInt64 & l, const Float64 & r) const { return l == r; }
bool operator() (const UInt64 & l, const Int64 & r) const { return r >= 0 && l == UInt64(r); }
bool operator() (const UInt64 & l, const Float64 & r) const { return Double128(l) == Double128(r); }
bool operator() (const UInt64 & l, const String & r) const { return l == stringToDateOrDateTime(r); }
bool operator() (const UInt64 & l, const Array & r) const { return false; }
bool operator() (const UInt64 & l, const Tuple & r) const { return false; }
bool operator() (const Int64 & l, const Null & r) const { return false; }
bool operator() (const Int64 & l, const UInt64 & r) const { return l == r; }
bool operator() (const Int64 & l, const UInt64 & r) const { return l >= 0 && UInt64(l) == r; }
bool operator() (const Int64 & l, const Int64 & r) const { return l == r; }
bool operator() (const Int64 & l, const Float64 & r) const { return l == r; }
bool operator() (const Int64 & l, const Float64 & r) const { return Double128(l) == Double128(r); }
bool operator() (const Int64 & l, const String & r) const { return false; }
bool operator() (const Int64 & l, const Array & r) const { return false; }
bool operator() (const Int64 & l, const Tuple & r) const { return false; }
bool operator() (const Float64 & l, const Null & r) const { return false; }
bool operator() (const Float64 & l, const UInt64 & r) const { return l == r; }
bool operator() (const Float64 & l, const Int64 & r) const { return l == r; }
bool operator() (const Float64 & l, const UInt64 & r) const { return Double128(l) == Double128(r); }
bool operator() (const Float64 & l, const Int64 & r) const { return Double128(l) == Double128(r); }
bool operator() (const Float64 & l, const Float64 & r) const { return l == r; }
bool operator() (const Float64 & l, const String & r) const { return false; }
bool operator() (const Float64 & l, const Array & r) const { return false; }
@ -324,6 +324,8 @@ public:
class FieldVisitorAccurateLess : public StaticVisitor<bool>
{
using Double128 = long double; /// Non portable. Must have 64 bit mantissa to provide accurate comparisons.
public:
bool operator() (const Null & l, const Null & r) const { return false; }
bool operator() (const Null & l, const UInt64 & r) const { return true; }
@ -335,23 +337,23 @@ public:
bool operator() (const UInt64 & l, const Null & r) const { return false; }
bool operator() (const UInt64 & l, const UInt64 & r) const { return l < r; }
bool operator() (const UInt64 & l, const Int64 & r) const { return l < r; }
bool operator() (const UInt64 & l, const Float64 & r) const { return l < r; }
bool operator() (const UInt64 & l, const Int64 & r) const { return r >= 0 && l < UInt64(r); }
bool operator() (const UInt64 & l, const Float64 & r) const { return Double128(l) < Double128(r); }
bool operator() (const UInt64 & l, const String & r) const { return l < stringToDateOrDateTime(r); }
bool operator() (const UInt64 & l, const Array & r) const { return true; }
bool operator() (const UInt64 & l, const Tuple & r) const { return true; }
bool operator() (const Int64 & l, const Null & r) const { return false; }
bool operator() (const Int64 & l, const UInt64 & r) const { return l < r; }
bool operator() (const Int64 & l, const UInt64 & r) const { return l < 0 || UInt64(l) < r; }
bool operator() (const Int64 & l, const Int64 & r) const { return l < r; }
bool operator() (const Int64 & l, const Float64 & r) const { return l < r; }
bool operator() (const Int64 & l, const Float64 & r) const { return Double128(l) < Double128(r); }
bool operator() (const Int64 & l, const String & r) const { return true; }
bool operator() (const Int64 & l, const Array & r) const { return true; }
bool operator() (const Int64 & l, const Tuple & r) const { return true; }
bool operator() (const Float64 & l, const Null & r) const { return false; }
bool operator() (const Float64 & l, const UInt64 & r) const { return l < r; }
bool operator() (const Float64 & l, const Int64 & r) const { return l < r; }
bool operator() (const Float64 & l, const UInt64 & r) const { return Double128(l) < Double128(r); }
bool operator() (const Float64 & l, const Int64 & r) const { return Double128(l) < Double128(r); }
bool operator() (const Float64 & l, const Float64 & r) const { return l < r; }
bool operator() (const Float64 & l, const String & r) const { return true; }
bool operator() (const Float64 & l, const Array & r) const { return true; }
@ -382,6 +384,4 @@ public:
bool operator() (const Tuple & l, const Tuple & r) const { return l < r; }
};
#pragma GCC diagnostic pop
}

67
dbms/include/DB/Core/Progress.h
View File

@ -11,20 +11,20 @@ namespace DB
{
/** Прогресс выполнения запроса.
* Передаваемые по сети значения представляют собой разницу - сколько было сделано после предыдущего отправленного значения.
* Тот же объект используется для суммирования полученных значений.
/** Progress of query execution.
* Values, transferred over network are deltas - how much was done after previously sent value.
* The same struct is also used for summarized values.
*/
struct Progress
{
size_t rows = 0; /// Строк обработано.
size_t bytes = 0; /// Байт обработано.
std::atomic<size_t> rows {0}; /// Rows (source) processed.
std::atomic<size_t> bytes {0}; /// Bytes (uncompressed, source) processed.
/** Сколько ещё строк надо обработать, приблизительно. Передаётся не ноль, когда возникает информация о какой-то новой части работы.
* Полученные значения надо суммровать, чтобы получить оценку общего количества строк для обработки.
* Используется для отображения прогресс-бара на клиенте.
/** How much rows must be processed, in total, approximately. Non-zero value is sent when there is information about some new part of job.
* Received values must be summed to get estimate of total rows to process.
* Used for rendering progress bar on client.
*/
size_t total_rows = 0;
std::atomic<size_t> total_rows {0};
Progress() {}
Progress(size_t rows_, size_t bytes_, size_t total_rows_ = 0)
@ -32,11 +32,19 @@ struct Progress
void read(ReadBuffer & in, UInt64 server_revision)
{
readVarUInt(rows, in);
readVarUInt(bytes, in);
size_t new_rows = 0;
size_t new_bytes = 0;
size_t new_total_rows = 0;
readVarUInt(new_rows, in);
readVarUInt(new_bytes, in);
if (server_revision >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS)
readVarUInt(total_rows, in);
readVarUInt(new_total_rows, in);
rows = new_rows;
bytes = new_bytes;
total_rows = new_total_rows;
}
void write(WriteBuffer & out, UInt64 client_revision) const
@ -48,36 +56,45 @@ struct Progress
writeVarUInt(total_rows, out);
}
void increment(const Progress & rhs)
/// Each value separately is changed atomically (but not whole object).
void incrementPiecewiseAtomically(const Progress & rhs)
{
rows += rhs.rows;
bytes += rhs.bytes;
total_rows += rhs.total_rows;
}
/// Каждое значение по-отдельности изменяется атомарно.
void incrementPiecewiseAtomically(const Progress & rhs)
{
__sync_add_and_fetch(&rows, rhs.rows);
__sync_add_and_fetch(&bytes, rhs.bytes);
__sync_add_and_fetch(&total_rows, rhs.total_rows);
}
void reset()
{
*this = Progress();
rows = 0;
bytes = 0;
total_rows = 0;
}
Progress fetchAndResetPiecewiseAtomically()
{
Progress res;
res.rows = __sync_fetch_and_and(&rows, 0);
res.bytes = __sync_fetch_and_and(&bytes, 0);
res.total_rows = __sync_fetch_and_and(&total_rows, 0);
res.rows = rows.fetch_and(0);
res.bytes = bytes.fetch_and(0);
res.total_rows = total_rows.fetch_and(0);
return res;
}
Progress & operator=(Progress && other)
{
rows = other.rows.load(std::memory_order_relaxed);
bytes = other.bytes.load(std::memory_order_relaxed);
total_rows = other.total_rows.load(std::memory_order_relaxed);
return *this;
}
Progress(Progress && other)
{
*this = std::move(other);
}
};

4
dbms/include/DB/Core/SortDescription.h
View File

@ -86,7 +86,7 @@ struct SortCursorImpl
size_t num_columns = block.columns();
for (size_t j = 0; j < num_columns; ++j)
all_columns.push_back(&*block.getByPosition(j).column);
all_columns.push_back(block.getByPosition(j).column.get());
for (size_t j = 0, size = desc.size(); j < size; ++j)
{
@ -94,7 +94,7 @@ struct SortCursorImpl
? block.getPositionByName(desc[j].column_name)
: desc[j].column_number;
sort_columns.push_back(&*block.getByPosition(column_number).column);
sort_columns.push_back(block.getByPosition(column_number).column.get());
need_collation[j] = desc[j].collator != nullptr && sort_columns.back()->getName() == "ColumnString";
has_collation |= need_collation[j];

14
dbms/include/DB/Core/StringRef.h
View File

@ -13,6 +13,8 @@
#include <emmintrin.h>
#endif
#include <DB/Common/unaligned.h>
/// Штука, чтобы не создавать строки для поиска подстроки в хэш таблице.
struct StringRef
@ -197,8 +199,8 @@ inline size_t hashLessThan8(const char * data, size_t size)
if (size >= 4)
{
uint64 a = *reinterpret_cast<const uint32_t *>(data);;
return hashLen16(size + (a << 3), *reinterpret_cast<const uint32_t *>(data + size - 4));
uint64 a = unalignedLoad<uint32_t>(data);
return hashLen16(size + (a << 3), unalignedLoad<uint32_t>(data + size - 4));
}
if (size > 0)
@ -218,8 +220,8 @@ inline size_t hashLessThan16(const char * data, size_t size)
{
if (size > 8)
{
uint64 a = *reinterpret_cast<const uint64_t *>(data);
uint64 b = *reinterpret_cast<const uint64_t *>(data + size - 8);
uint64 a = unalignedLoad<uint64_t>(data);
uint64 b = unalignedLoad<uint64_t>(data + size - 8);
return hashLen16(a, rotateByAtLeast1(b + size, size)) ^ b;
}
@ -246,13 +248,13 @@ struct CRC32Hash
do
{
uint64_t word = *reinterpret_cast<const uint64_t *>(pos);
uint64_t word = unalignedLoad<uint64_t>(pos);
res = _mm_crc32_u64(res, word);
pos += 8;
} while (pos + 8 < end);
uint64_t word = *reinterpret_cast<const uint64_t *>(end - 8); /// Не уверен, что это нормально.
uint64_t word = unalignedLoad<uint64_t>(end - 8); /// Не уверен, что это нормально.
res = _mm_crc32_u64(res, word);
return res;

3
dbms/include/DB/DataStreams/AddingConstColumnBlockInputStream.h
View File

@ -39,8 +39,7 @@ protected:
if (!res)
return res;
ColumnPtr column_ptr = ColumnConst<ColumnType>(res.rows(), value, data_type).convertToFullColumn();
ColumnWithTypeAndName column(column_ptr, data_type, column_name);
res.insert(column);
res.insert({column_ptr, data_type, column_name});
return res;
}

5
dbms/include/DB/DataStreams/AsynchronousBlockInputStream.h
View File

@ -1,12 +1,11 @@
#pragma once
#include <threadpool.hpp>
#include <Poco/Event.h>
#include <DB/DataStreams/IProfilingBlockInputStream.h>
#include <DB/Common/setThreadName.h>
#include <DB/Common/CurrentMetrics.h>
#include <DB/Common/ThreadPool.h>
namespace DB
@ -81,7 +80,7 @@ public:
}
protected:
boost::threadpool::pool pool{1};
ThreadPool pool{1};
Poco::Event ready;
bool started = false;
bool first = true;

9
dbms/include/DB/DataStreams/BinaryRowOutputStream.h
View File

@ -1,13 +1,16 @@
#pragma once
#include <DB/Core/Block.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/DataStreams/IRowOutputStream.h>
namespace DB
{
class IColumn;
class IDataType;
class WriteBuffer;
/** Поток для вывода данных в бинарном построчном формате.
*/
class BinaryRowOutputStream : public IRowOutputStream
@ -17,7 +20,7 @@ public:
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void flush() override { ostr.next(); }
void flush() override;
String getContentType() const override { return "application/octet-stream"; }

2
dbms/include/DB/DataStreams/BlockIO.h
View File

@ -39,8 +39,6 @@ struct BlockIO
{
if (exception_callback)
exception_callback();
else
tryLogCurrentException(__PRETTY_FUNCTION__);
}
BlockIO & operator= (const BlockIO & rhs)

1
dbms/include/DB/DataStreams/BlockOutputStreamFromRowOutputStream.h
View File

@ -23,6 +23,7 @@ public:
void setRowsBeforeLimit(size_t rows_before_limit) override;
void setTotals(const Block & totals) override;
void setExtremes(const Block & extremes) override;
void onProgress(const Progress & progress) override;
String getContentType() const override { return row_output->getContentType(); }

Some files were not shown because too many files have changed in this diff Show More