thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

merged with master

Browse Source
This commit is contained in:
Nikolai Kochetov 2018-04-16 18:35:11 +03:00
commit 9e05052c2a
564 changed files with 8680 additions and 5365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "contrib/zookeeper"]
path = contrib/zookeeper
url = https://github.com/ClickHouse-Extras/zookeeper.git
[submodule "contrib/poco"]
path = contrib/poco
url = https://github.com/ClickHouse-Extras/poco

2
CHANGELOG.md
View File

@ -1,4 +1,4 @@
# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
# ClickHouse release 1.1.54370, 2018-03-16
## New features:

2
CHANGELOG_RU.md
View File

@ -1,4 +1,4 @@
# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
# ClickHouse release 1.1.54370, 2018-03-16
## Новые возможности:

2
CMakeLists.txt
View File

@ -252,13 +252,13 @@ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake)
else ()
include (cmake/find_odbc.cmake)
endif ()
message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}")
include (cmake/find_poco.cmake)
include (cmake/find_lz4.cmake)
include (cmake/find_sparsehash.cmake)
include (cmake/find_rt.cmake)
include (cmake/find_execinfo.cmake)
include (cmake/find_readline_edit.cmake)
include (cmake/find_zookeeper.cmake)
include (cmake/find_re2.cmake)
include (cmake/find_rdkafka.cmake)
include (cmake/find_capnp.cmake)

66
CONTRIBUTING.md
View File

@ -1,32 +1,34 @@
# Contributing to ClickHouse
## Technical info
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
https://clickhouse.yandex/docs/en/development/index.html
## Legal info
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
1) https://yandex.ru/legal/cla/?lang=en (in English) and
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
By adopting the CLA, you state the following:
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
* You has read the terms and conditions of the CLA and agree with them in full,
* You are legally able to provide and license your contributions as stated,
* We may use your contributions for our open source projects and for any other our project too,
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
```
I hereby agree to the terms of the CLA available at: [link].
```
Replace the bracketed text as follows:
* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).
It is enough to provide us such notification at once.
# Contributing to ClickHouse
## Technical info
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
https://clickhouse.yandex/docs/en/development/index.html
## Legal info
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
1) https://yandex.ru/legal/cla/?lang=en (in English) and
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
By adopting the CLA, you state the following:
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
* You has read the terms and conditions of the CLA and agree with them in full,
* You are legally able to provide and license your contributions as stated,
* We may use your contributions for our open source projects and for any other our project too,
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
```
I hereby agree to the terms of the CLA available at: [link].
```
Replace the bracketed text as follows:
* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).
It is enough to provide us such notification at once.
If you don't agree with the CLA, you still can open a pull request to provide your contributions.

3
README.md
View File

@ -1,5 +1,6 @@
# ClickHouse
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
[Read more...](https://clickhouse.yandex/)
Learn more about ClickHouse at [https://clickhouse.yandex/](https://clickhouse.yandex/)
[![Build Status](https://travis-ci.org/yandex/ClickHouse.svg?branch=master)](https://travis-ci.org/yandex/ClickHouse)

3
cmake/Modules/FindPoco.cmake
View File

@ -51,6 +51,7 @@
set(Poco_HINTS
/usr/local
/usr/local/include/Poco
C:/AppliedInformatics
${Poco_DIR}
$ENV{Poco_DIR}
@ -230,5 +231,3 @@ if(${Poco_OSP_FOUND})
endif()
message(STATUS "Found Poco: ${Poco_LIBRARIES}")

50
cmake/find_poco.cmake
View File

@ -9,7 +9,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt")
endif ()
if (NOT USE_INTERNAL_POCO_LIBRARY)
find_package (Poco COMPONENTS Net NetSSL XML Data Crypto DataODBC MongoDB)
find_package (Poco COMPONENTS Net NetSSL XML SQL Data Crypto DataODBC MongoDB)
endif ()
if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
@ -24,6 +24,15 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (ENABLE_DATA_SQLITE 0 CACHE BOOL "")
set (ENABLE_DATA_MYSQL 0 CACHE BOOL "")
set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "")
# new after 2.0.0:
set (POCO_ENABLE_ZIP 0 CACHE BOOL "")
set (POCO_ENABLE_PAGECOMPILER 0 CACHE BOOL "")
set (POCO_ENABLE_PAGECOMPILER_FILE2PAGE 0 CACHE BOOL "")
set (POCO_ENABLE_REDIS 0 CACHE BOOL "")
set (POCO_ENABLE_SQL_SQLITE 0 CACHE BOOL "")
set (POCO_ENABLE_SQL_MYSQL 0 CACHE BOOL "")
set (POCO_ENABLE_SQL_POSTGRESQL 0 CACHE BOOL "")
set (POCO_UNBUNDLED 1 CACHE BOOL "")
set (POCO_UNBUNDLED_PCRE 0 CACHE BOOL "")
set (POCO_UNBUNDLED_EXPAT 0 CACHE BOOL "")
@ -43,13 +52,39 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (Poco_MongoDB_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/")
endif ()
if (ODBC_FOUND)
set (Poco_DataODBC_FOUND 1)
set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
if (EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/")
set (Poco_SQL_FOUND 1)
set (Poco_SQL_LIBRARY PocoSQL)
set (Poco_SQL_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/include"
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include"
)
if (ODBC_FOUND)
set (Poco_SQLODBC_FOUND 1)
set (Poco_SQLODBC_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/"
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
${ODBC_INCLUDE_DIRECTORIES}
)
set (Poco_SQLODBC_LIBRARY PocoSQLODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
endif ()
else ()
set (Poco_Data_FOUND 1)
set (Poco_Data_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include")
set (Poco_Data_LIBRARY PocoData)
if (ODBC_FOUND)
set (Poco_DataODBC_FOUND 1)
set (Poco_DataODBC_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
${ODBC_INCLUDE_DIRECTORIES}
)
set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
endif ()
endif ()
if (OPENSSL_FOUND)
# TODO! fix internal ssl
if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
set (Poco_NetSSL_FOUND 1)
set (Poco_NetSSL_LIBRARY PocoNetSSL)
set (Poco_Crypto_LIBRARY PocoCrypto)
@ -65,7 +100,6 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (Poco_Foundation_LIBRARY PocoFoundation)
set (Poco_Util_LIBRARY PocoUtil)
set (Poco_Net_LIBRARY PocoNet)
set (Poco_Data_LIBRARY PocoData)
set (Poco_XML_LIBRARY PocoXML)
endif ()
@ -85,3 +119,5 @@ message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${
# ClickHouse-Extras/clickhouse_warning
# ClickHouse-Extras/clickhouse-purge-logs-on-no-space
# ClickHouse-Extras/clickhouse_freebsd
# ClickHouse-Extras/clikhouse_no_zlib
# ClickHouse-Extras/clickhouse-fix-atomic

35
cmake/find_zookeeper.cmake
View File

@ -1,35 +0,0 @@
if (ARCH_ARM)
# bundled have some asm broken for arm, use package libzookeeper-mt-dev
set(USE_INTERNAL_ZOOKEEPER_LIBRARY 0 CACHE BOOL "")
endif ()
option (USE_INTERNAL_ZOOKEEPER_LIBRARY "Set to FALSE to use system zookeeper library instead of bundled" ${NOT_UNBUNDLED})
if (USE_INTERNAL_ZOOKEEPER_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zookeeper/src/c/CMakeLists.txt")
message (WARNING "submodule contrib/zookeeper is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_ZOOKEEPER_LIBRARY 0)
endif ()
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
find_library (ZOOKEEPER_LIBRARY zookeeper_mt)
find_path (ZOOKEEPER_INCLUDE_DIR NAMES zookeeper/zookeeper.h PATHS ${ZOOKEEPER_INCLUDE_PATHS})
set(ZOOKEEPER_INCLUDE_DIR "${ZOOKEEPER_INCLUDE_DIR}/zookeeper")
endif ()
if (ZOOKEEPER_LIBRARY AND ZOOKEEPER_INCLUDE_DIR)
else ()
set (USE_INTERNAL_ZOOKEEPER_LIBRARY 1)
set(WANT_CPPUNIT 0 CACHE BOOL "")
set (ZOOKEEPER_LIBRARY zookeeper)
endif ()
message (STATUS "Using zookeeper: ${ZOOKEEPER_INCLUDE_DIR} : ${ZOOKEEPER_LIBRARY}")
# how to make clickhouse branch of https://github.com/ClickHouse-Extras/zookeeper.git :
# git remote add upstream https://github.com/apache/zookeeper.git
# git checkhout upstream/master
# git branch -D clickhouse
# git checkout -b clickhouse
# git merge clickhouse_misc
# git merge clickhouse_706

24
contrib/CMakeLists.txt vendored
View File

@ -25,10 +25,6 @@ if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
add_subdirectory (double-conversion)
endif ()
if (USE_INTERNAL_ZOOKEEPER_LIBRARY)
add_subdirectory (zookeeper/src/c)
endif ()
if (USE_INTERNAL_CITYHASH_LIBRARY)
add_subdirectory (libcityhash)
endif ()
@ -100,13 +96,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
mark_as_advanced (ZLIB_INCLUDE_DIR)
if (USE_INTERNAL_SSL_LIBRARY)
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
set (WITH_BUNDLED_SSL 1)
if (MAKE_STATIC_LIBRARIES)
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
else ()
set (WITH_SSL 0 CACHE INTERNAL "")
endif ()
endif ()
add_subdirectory (librdkafka)
if (USE_INTERNAL_SSL_LIBRARY)
if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
endif ()
target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
@ -127,16 +127,18 @@ endif ()
if (USE_INTERNAL_POCO_LIBRARY)
set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
set (_save ${ENABLE_TESTS})
set (ENABLE_TESTS 0)
set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
if (USE_INTERNAL_SSL_LIBRARY)
set (DISABLE_INTERNAL_OPENSSL 1)
set (ENABLE_NETSSL 0) # TODO!
set (ENABLE_CRYPTO 0) # TODO!
set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
endif ()
if (MSVC)
set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
endif ()
add_subdirectory (poco)
unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)

2
contrib/boost vendored

@ -1 +1 @@
Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e
Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded

4
contrib/libcityhash/src/config.h
View File

@ -5,7 +5,11 @@
/* #undef AC_APPLE_UNIVERSAL_BUILD */
/* Define to 1 if the compiler supports __builtin_expect. */
#if _MSC_VER
#define HAVE_BUILTIN_EXPECT 0
#else
#define HAVE_BUILTIN_EXPECT 1
#endif
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

4
contrib/libfarmhash/CMakeLists.txt
View File

@ -2,4 +2,8 @@ add_library(farmhash
farmhash.cc
farmhash.h)
if (MSVC)
target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1)
endif ()
target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

4
contrib/libtcmalloc/src/config.h
View File

@ -13,7 +13,11 @@
/* #undef ENABLE_SIZED_DELETE */
/* Define to 1 if compiler supports __builtin_expect */
#if _MSC_VER
#define HAVE_BUILTIN_EXPECT 0
#else
#define HAVE_BUILTIN_EXPECT 1
#endif
/* Define to 1 if compiler supports __builtin_stack_pointer */
/* #undef HAVE_BUILTIN_STACK_POINTER */

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit 8238852d7ab2a4abdf87adff233b3b83686f4fe4
Subproject commit 930a7ec1154f4f9711edfb4b4a39f9fff2a5bbb5

1
contrib/zookeeper vendored

@ -1 +0,0 @@
Subproject commit 438afae5af36c5be9c82d074f43a9bb19e0797c0

2
contrib/zstd vendored

@ -1 +1 @@
Subproject commit f4340f46b2387bc8de7d5320c0b83bb1499933ad
Subproject commit 255597502c3a4ef150abc964e376d4202a8c2929

2
copy_headers.sh
View File

@ -8,7 +8,7 @@
# Требуется clang, желательно наиболее свежий (trunk).
#
# Используется при сборке пакетов.
# Заголовочные файлы записываются в пакет clickhouse-server-base, в директорию /usr/share/clickhouse/headers.
# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers.
#
# Если вы хотите установить их самостоятельно, без сборки пакета,
# чтобы clickhouse-server видел их там, где ожидается, выполните:

32
dbms/CMakeLists.txt
View File

@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io
${LINK_LIBRARIES_ONLY_ON_X86_64}
${LZ4_LIBRARY}
${ZSTD_LIBRARY}
${ZOOKEEPER_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
${Poco_Net_LIBRARY}
${Poco_Data_LIBRARY}
@ -146,8 +145,6 @@ target_link_libraries (dbms
clickhouse_common_config
clickhouse_common_io
${MYSQLXX_LIBRARY}
${FARMHASH_LIBRARIES}
${METROHASH_LIBRARIES}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
${OPENSSL_CRYPTO_LIBRARY}
@ -158,19 +155,33 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_BOOST_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
endif ()
if (Poco_DataODBC_FOUND)
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES})
if (Poco_SQLODBC_FOUND)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
endif()
endif()
if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
endif()
if (Poco_DataODBC_FOUND)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
endif()
endif()
if (Poco_MongoDB_FOUND)
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
endif()
@ -217,6 +228,7 @@ endif ()
target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
# also for copy_headers.sh:

5
dbms/cmake/version.cmake
View File

@ -1,6 +1,7 @@
# This strings autochanged from release_lib.sh:
set(VERSION_DESCRIBE v1.1.54371-testing)
set(VERSION_REVISION 54371)
set(VERSION_DESCRIBE v1.1.54378-testing)
set(VERSION_REVISION 54378)
set(VERSION_GITHASH 5b19d89133a5ff7c72e40cc8c0226cb00466ba10)
# end of autochange
set (VERSION_MAJOR 1)

50
dbms/src/Client/Connection.cpp
View File

@ -12,6 +12,7 @@
#include <DataStreams/NativeBlockInputStream.h>
#include <DataStreams/NativeBlockOutputStream.h>
#include <Client/Connection.h>
#include <Client/TimeoutSetter.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Exception.h>
#include <Common/NetException.h>
@ -50,14 +51,15 @@ void Connection::connect()
if (connected)
disconnect();
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user);
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
<< (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
if (static_cast<bool>(encryption))
if (static_cast<bool>(secure))
{
#if Poco_NetSSL_FOUND
socket = std::make_unique<Poco::Net::SecureStreamSocket>();
#else
throw Exception{"tcp_ssl protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
throw Exception{"tcp_secure protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
#endif
}
else
@ -148,6 +150,10 @@ void Connection::receiveHello()
{
readStringBinary(server_timezone, *in);
}
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
{
readStringBinary(server_display_name, *in);
}
}
else if (packet_type == Protocol::Server::Exception)
receiveException()->rethrow();
@ -203,6 +209,14 @@ const String & Connection::getServerTimezone()
return server_timezone;
}
const String & Connection::getServerDisplayName()
{
if (!connected)
connect();
return server_display_name;
}
void Connection::forceConnected()
{
if (!connected)
@ -216,37 +230,11 @@ void Connection::forceConnected()
}
}
struct TimeoutSetter
{
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
: socket(socket_), timeout(timeout_)
{
old_send_timeout = socket.getSendTimeout();
old_receive_timeout = socket.getReceiveTimeout();
if (old_send_timeout > timeout)
socket.setSendTimeout(timeout);
if (old_receive_timeout > timeout)
socket.setReceiveTimeout(timeout);
}
~TimeoutSetter()
{
socket.setSendTimeout(old_send_timeout);
socket.setReceiveTimeout(old_receive_timeout);
}
Poco::Net::StreamSocket & socket;
Poco::Timespan timeout;
Poco::Timespan old_send_timeout;
Poco::Timespan old_receive_timeout;
};
bool Connection::ping()
{
// LOG_TRACE(log_wrapper.get(), "Ping");
TimeoutSetter timeout_setter(*socket, sync_request_timeout);
TimeoutSetter timeout_setter(*socket, sync_request_timeout, true);
try
{
UInt64 pong = 0;
@ -286,7 +274,7 @@ TablesStatusResponse Connection::getTablesStatus(const TablesStatusRequest & req
if (!connected)
connect();
TimeoutSetter timeout_setter(*socket, sync_request_timeout);
TimeoutSetter timeout_setter(*socket, sync_request_timeout, true);
writeVarUInt(Protocol::Client::TablesStatusRequest, *out);
request.write(*out, server_revision);

12
dbms/src/Client/Connection.h
View File

@ -58,14 +58,14 @@ public:
const ConnectionTimeouts & timeouts_,
const String & client_name_ = "client",
Protocol::Compression compression_ = Protocol::Compression::Enable,
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
Protocol::Secure secure_ = Protocol::Secure::Disable,
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
:
host(host_), port(port_), default_database(default_database_),
user(user_), password(password_), resolved_address(host, port),
client_name(client_name_),
compression(compression_),
encryption(encryption_),
secure(secure_),
timeouts(timeouts_),
sync_request_timeout(sync_request_timeout_),
log_wrapper(*this)
@ -84,7 +84,7 @@ public:
const ConnectionTimeouts & timeouts_,
const String & client_name_ = "client",
Protocol::Compression compression_ = Protocol::Compression::Enable,
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
Protocol::Secure secure_ = Protocol::Secure::Disable,
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
:
host(host_), port(port_),
@ -93,7 +93,7 @@ public:
resolved_address(resolved_address_),
client_name(client_name_),
compression(compression_),
encryption(encryption_),
secure(secure_),
timeouts(timeouts_),
sync_request_timeout(sync_request_timeout_),
log_wrapper(*this)
@ -134,6 +134,7 @@ public:
void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
const String & getServerTimezone();
const String & getServerDisplayName();
/// For log and exception messages.
const String & getDescription() const;
@ -213,6 +214,7 @@ private:
UInt64 server_version_minor = 0;
UInt64 server_revision = 0;
String server_timezone;
String server_display_name;
std::unique_ptr<Poco::Net::StreamSocket> socket;
std::shared_ptr<ReadBuffer> in;
@ -220,7 +222,7 @@ private:
String query_id;
Protocol::Compression compression; /// Enable data compression for communication.
Protocol::Encryption encryption; /// Enable data encryption for communication.
Protocol::Secure secure; /// Enable data encryption for communication.
/// What compression settings to use while sending data for INSERT queries and external tables.
CompressionSettings compression_settings;

14
dbms/src/Client/ConnectionPool.h
View File

@ -51,11 +51,12 @@ public:
const ConnectionTimeouts & timeouts,
const String & client_name_ = "client",
Protocol::Compression compression_ = Protocol::Compression::Enable,
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
Protocol::Secure secure_ = Protocol::Secure::Disable)
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
host(host_), port(port_), default_database(default_database_),
user(user_), password(password_), resolved_address(host_, port_),
client_name(client_name_), compression(compression_), encryption(encryption_),
client_name(client_name_), compression(compression_),
secure{secure_},
timeouts(timeouts)
{
}
@ -67,11 +68,12 @@ public:
const ConnectionTimeouts & timeouts,
const String & client_name_ = "client",
Protocol::Compression compression_ = Protocol::Compression::Enable,
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
Protocol::Secure secure_ = Protocol::Secure::Disable)
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
host(host_), port(port_), default_database(default_database_),
user(user_), password(password_), resolved_address(resolved_address_),
client_name(client_name_), compression(compression_), encryption(encryption_),
client_name(client_name_), compression(compression_),
secure{secure_},
timeouts(timeouts)
{
}
@ -102,7 +104,7 @@ protected:
return std::make_shared<Connection>(
host, port, resolved_address,
default_database, user, password, timeouts,
client_name, compression, encryption);
client_name, compression, secure);
}
private:
@ -119,7 +121,7 @@ private:
String client_name;
Protocol::Compression compression; /// Whether to compress data when interacting with the server.
Protocol::Encryption encryption; /// Whether to encrypt data when interacting with the server.
Protocol::Secure secure; /// Whether to encrypt data when interacting with the server.
ConnectionTimeouts timeouts;
};

6
dbms/src/Client/MultiplexedConnections.cpp
View File

@ -280,8 +280,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
read_list.push_back(*connection->socket);
}
/// If no data was found, then we check if there are any connections
/// ready for reading.
/// If no data was found, then we check if there are any connections ready for reading.
if (read_list.empty())
{
Poco::Net::Socket::SocketList write_list;
@ -300,6 +299,9 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
throw Exception("Timeout exceeded while reading from " + dumpAddressesUnlocked(), ErrorCodes::TIMEOUT_EXCEEDED);
}
/// TODO Absolutely wrong code: read_list could be empty; rand() is not thread safe and has low quality; motivation of rand is unclear.
/// This code path is disabled by default.
auto & socket = read_list[rand() % read_list.size()];
if (fd_to_replica_state_idx.empty())
{

47
dbms/src/Client/TimeoutSetter.h Normal file
View File

@ -0,0 +1,47 @@
#pragma once
#include <Poco/Timespan.h>
#include <Poco/Net/StreamSocket.h>
namespace DB
{
/// Temporarily overrides socket send/recieve timeouts and reset them back into destructor
/// If "limit_max_timeout" is true, timeouts could be only decreased (maxed by previous value).
struct TimeoutSetter
{
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & send_timeout_, const Poco::Timespan & recieve_timeout_,
bool limit_max_timeout = false)
: socket(socket_), send_timeout(send_timeout_), recieve_timeout(recieve_timeout_)
{
old_send_timeout = socket.getSendTimeout();
old_receive_timeout = socket.getReceiveTimeout();
if (!limit_max_timeout || old_send_timeout > send_timeout)
socket.setSendTimeout(send_timeout);
if (!limit_max_timeout || old_receive_timeout > recieve_timeout)
socket.setReceiveTimeout(recieve_timeout);
}
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_, bool limit_max_timeout = false)
: TimeoutSetter(socket_, timeout_, timeout_, limit_max_timeout) {}
~TimeoutSetter()
{
socket.setSendTimeout(old_send_timeout);
socket.setReceiveTimeout(old_receive_timeout);
}
Poco::Net::StreamSocket & socket;
Poco::Timespan send_timeout;
Poco::Timespan recieve_timeout;
Poco::Timespan old_send_timeout;
Poco::Timespan old_receive_timeout;
};
}

6
dbms/src/Columns/ColumnAggregateFunction.cpp
View File

@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start
}
MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
{
size_t size = getData().size();
if (size != filter.size())
@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t
}
MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
{
size_t size = getData().size();
@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n)
data.resize_assume_reserved(new_size);
}
MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
{
size_t size = data.size();
if (size != offsets.size())

6
dbms/src/Columns/ColumnAggregateFunction.h
View File

@ -152,11 +152,11 @@ public:
void popBack(size_t n) override;
MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
MutableColumnPtr replicate(const Offsets & offsets) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;

58
dbms/src/Columns/ColumnArray.cpp
View File

@ -31,10 +31,10 @@ namespace ErrorCodes
}
ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
: data(nested_column), offsets(offsets_column)
ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
: data(std::move(nested_column)), offsets(std::move(offsets_column))
{
if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
if (!typeid_cast<const ColumnOffsets *>(offsets.get()))
throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
/** NOTE
@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs
*/
}
ColumnArray::ColumnArray(const ColumnPtr & nested_column)
: data(nested_column)
ColumnArray::ColumnArray(MutableColumnPtr && nested_column)
: data(std::move(nested_column))
{
if (!data->empty())
throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN);
@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
}
MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const
ColumnPtr ColumnArray::convertToFullColumnIfConst() const
{
ColumnPtr new_data;
@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
}
MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
{
if (typeid_cast<const ColumnUInt8 *>(data.get())) return filterNumber<UInt8>(filt, result_size_hint);
if (typeid_cast<const ColumnUInt16 *>(data.get())) return filterNumber<UInt16>(filt, result_size_hint);
@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi
}
template <typename T>
MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return ColumnArray::create(data);
@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s
return std::move(res);
}
MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
{
size_t col_size = getOffsets().size();
if (col_size != filt.size())
@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s
return std::move(res);
}
MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
{
size_t size = getOffsets().size();
if (size != filt.size())
@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_
return std::move(res);
}
MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return ColumnArray::create(data);
@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result
filtered_offsets);
}
MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return ColumnArray::create(data);
@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
Columns temporary_arrays(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint);
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
.filter(filt, result_size_hint);
Columns tuple_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
}
MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
{
size_t size = getOffsets().size();
@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
}
MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
{
if (typeid_cast<const ColumnUInt8 *>(data.get())) return replicateNumber<UInt8>(replicate_offsets);
if (typeid_cast<const ColumnUInt16 *>(data.get())) return replicateNumber<UInt16>(replicate_offsets);
@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
template <typename T>
MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets)
}
MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets)
}
MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets)
}
MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets
}
MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
{
const ColumnNullable & nullable = static_cast<const ColumnNullable &>(*data);
/// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result.
/// NOTE Offsets are calculated twice and it is redundant.
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
.replicate(replicate_offsets);
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
.replicate(replicate_offsets);
return ColumnArray::create(
ColumnNullable::create(
static_cast<ColumnArray &>(*array_of_nested).getDataPtr(),
static_cast<ColumnArray &>(*array_of_null_map).getDataPtr()),
static_cast<ColumnArray &>(*array_of_nested).getOffsetsPtr());
static_cast<const ColumnArray &>(*array_of_nested).getDataPtr(),
static_cast<const ColumnArray &>(*array_of_null_map).getDataPtr()),
static_cast<const ColumnArray &>(*array_of_nested).getOffsetsPtr());
}
MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
{
const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets)
Columns temporary_arrays(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets);
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
.replicate(replicate_offsets);
Columns tuple_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)

52
dbms/src/Columns/ColumnArray.h
View File

@ -24,14 +24,32 @@ private:
friend class COWPtrHelper<IColumn, ColumnArray>;
/** Create an array column with specified values and offsets. */
ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column);
ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
/** Create an empty column of arrays with the type of values as in the column `nested_column` */
ColumnArray(const ColumnPtr & nested_column);
explicit ColumnArray(MutableColumnPtr && nested_column);
ColumnArray(const ColumnArray &) = default;
public:
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/
using Base = COWPtrHelper<IColumn, ColumnArray>;
static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
{
return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
}
static Ptr create(const ColumnPtr & nested_column)
{
return ColumnArray::create(nested_column->assumeMutable());
}
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
/** On the index i there is an offset to the beginning of the i + 1 -th element. */
using ColumnOffsets = ColumnVector<Offset>;
@ -51,15 +69,15 @@ public:
void insertFrom(const IColumn & src_, size_t n) override;
void insertDefault() override;
void popBack(size_t n) override;
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
MutableColumnPtr convertToFullColumnIfConst() const override;
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
ColumnPtr convertToFullColumnIfConst() const override;
void getExtremes(Field & min, Field & max) const override;
bool hasEqualOffsets(const ColumnArray & other) const;
@ -110,33 +128,33 @@ private:
/// Multiply values if the nested column is ColumnVector<T>.
template <typename T>
MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
ColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
/// Multiply the values if the nested column is ColumnString. The code is too complicated.
MutableColumnPtr replicateString(const Offsets & replicate_offsets) const;
ColumnPtr replicateString(const Offsets & replicate_offsets) const;
/** Non-constant arrays of constant values are quite rare.
* Most functions can not work with them, and does not create such columns as a result.
* An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions.
* Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst).
*/
MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const;
ColumnPtr replicateConst(const Offsets & replicate_offsets) const;
/** The following is done by simply replicating of nested columns.
*/
MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
ColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
ColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
/// Specializations for the filter function.
template <typename T>
MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
};

8
dbms/src/Columns/ColumnConst.cpp
View File

@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s)
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
}
MutableColumnPtr ColumnConst::convertToFullColumn() const
ColumnPtr ColumnConst::convertToFullColumn() const
{
return data->replicate(Offsets(1, s));
}
MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
{
if (s != filt.size())
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")",
@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_
return ColumnConst::create(data, countBytesInFilter(filt));
}
MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
{
if (s != offsets.size())
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")",
@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
return ColumnConst::create(data, replicated_size);
}
MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
{
if (limit == 0)
limit = s;

10
dbms/src/Columns/ColumnConst.h
View File

@ -29,9 +29,9 @@ private:
ColumnConst(const ColumnConst & src) = default;
public:
MutableColumnPtr convertToFullColumn() const;
ColumnPtr convertToFullColumn() const;
MutableColumnPtr convertToFullColumnIfConst() const override
ColumnPtr convertToFullColumnIfConst() const override
{
return convertToFullColumn();
}
@ -145,9 +145,9 @@ public:
data->updateHashWithValue(0, hash);
}
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr replicate(const Offsets & offsets) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
size_t byteSize() const override

6
dbms/src/Columns/ColumnFixedString.cpp
View File

@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
}
MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
{
size_t col_size = size();
if (col_size != filt.size())
@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t
return std::move(res);
}
MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
{
size_t col_size = size();
@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim
return std::move(res);
}
MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
{
size_t col_size = size();
if (col_size != offsets.size())

6
dbms/src/Columns/ColumnFixedString.h
View File

@ -104,11 +104,11 @@ public:
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
MutableColumnPtr replicate(const Offsets & offsets) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
{

8
dbms/src/Columns/ColumnFunction.cpp
View File

@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
return ColumnFunction::create(size, function, capture);
}
MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
{
if (size_ != offsets.size())
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column ("
@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
return ColumnFunction::create(replicated_size, function, capture);
}
MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
{
ColumnsWithTypeAndName capture = captured_columns;
for (auto & column : capture)
@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
return ColumnFunction::create(length, function, capture);
}
MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
{
if (size_ != filt.size())
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column ("
@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size
return ColumnFunction::create(filtered_size, function, capture);
}
MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
{
if (limit == 0)
limit = size_;

8
dbms/src/Columns/ColumnFunction.h
View File

@ -29,10 +29,10 @@ public:
size_t size() const override { return size_; }
MutableColumnPtr cut(size_t start, size_t length) const override;
MutableColumnPtr replicate(const Offsets & offsets) const override;
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr cut(size_t start, size_t length) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
void insertDefault() override;
void popBack(size_t n) override;
std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex num_columns,

14
dbms/src/Columns/ColumnNullable.cpp
View File

@ -18,8 +18,8 @@ namespace ErrorCodes
}
ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
: nested_column{nested_column_}, null_map{null_map_}
ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_)
: nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
{
/// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
{
ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
auto new_null_map = ColumnUInt8::create();
if (new_size > 0)
@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
memset(&new_null_map->getData()[count], 1, new_size - count);
}
return ColumnNullable::create(new_nested_col, std::move(new_null_map));
return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map));
}
@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n)
getNullMapColumn().popBack(n);
}
MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
{
ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint);
ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint);
return ColumnNullable::create(filtered_data, filtered_null_map);
}
MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
{
ColumnPtr permuted_data = getNestedColumn().permute(perm, limit);
ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit);
@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
}
MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
{
ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);

20
dbms/src/Columns/ColumnNullable.h
View File

@ -23,10 +23,22 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
private:
friend class COWPtrHelper<IColumn, ColumnNullable>;
ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_);
ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
ColumnNullable(const ColumnNullable &) = default;
public:
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/
using Base = COWPtrHelper<IColumn, ColumnNullable>;
static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
{
return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
}
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
const char * getFamilyName() const override { return "Nullable"; }
std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
MutableColumnPtr cloneResized(size_t size) const override;
@ -50,14 +62,14 @@ public:
}
void popBack(size_t n) override;
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
void updateHashWithValue(size_t n, SipHash & hash) const override;
void getExtremes(Field & min, Field & max) const override;

6
dbms/src/Columns/ColumnString.cpp
View File

@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
}
MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
{
if (offsets.size() == 0)
return ColumnString::create();
@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h
}
MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
{
size_t size = offsets.size();
@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
}
MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())

18
dbms/src/Columns/ColumnString.h
View File

@ -89,6 +89,12 @@ public:
return StringRef(&chars[offsetAt(n)], sizeAt(n));
}
/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
void insert(const Field & x) override
{
const String & s = DB::get<const String &>(x);
@ -101,7 +107,11 @@ public:
offsets.push_back(new_size);
}
void insertFrom(const IColumn & src_, size_t n) override
#if !__clang__
#pragma GCC diagnostic pop
#endif
void insertFrom(const IColumn & src_, size_t n) override
{
const ColumnString & src = static_cast<const ColumnString &>(src_);
@ -206,9 +216,9 @@ public:
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
void insertDefault() override
{
@ -239,7 +249,7 @@ public:
/// Sorting with respect of collation.
void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
{

33
dbms/src/Columns/ColumnTuple.cpp
View File

@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const
return res.str();
}
ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns)
ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
{
columns.reserve(mutable_columns.size());
for (auto & column : mutable_columns)
{
if (column->isColumnConst())
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
columns.push_back(std::move(column));
}
}
ColumnTuple::Ptr ColumnTuple::create(const Columns & columns)
{
for (const auto & column : columns)
if (column->isColumnConst())
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
auto column_tuple = ColumnTuple::create(MutableColumns());
column_tuple->columns = columns;
return std::move(column_tuple);
}
MutableColumnPtr ColumnTuple::cloneEmpty() const
{
const size_t tuple_size = columns.size();
Columns new_columns(tuple_size);
MutableColumns new_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
new_columns[i] = columns[i]->cloneEmpty();
return ColumnTuple::create(new_columns);
return ColumnTuple::create(std::move(new_columns));
}
Field ColumnTuple::operator[](size_t n) const
@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng
start, length);
}
MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
{
const size_t tuple_size = columns.size();
Columns new_columns(tuple_size);
@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi
return ColumnTuple::create(new_columns);
}
MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
{
const size_t tuple_size = columns.size();
Columns new_columns(tuple_size);
@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co
return ColumnTuple::create(new_columns);
}
MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
{
const size_t tuple_size = columns.size();
Columns new_columns(tuple_size);
@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se
for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
{
Columns new_columns(tuple_size);
MutableColumns new_columns(tuple_size);
for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx)
new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
res[scattered_idx] = ColumnTuple::create(new_columns);
res[scattered_idx] = ColumnTuple::create(std::move(new_columns));
}
return res;

17
dbms/src/Columns/ColumnTuple.h
View File

@ -22,10 +22,19 @@ private:
template <bool positive>
struct Less;
ColumnTuple(const Columns & columns);
explicit ColumnTuple(MutableColumns && columns);
ColumnTuple(const ColumnTuple &) = default;
public:
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/
using Base = COWPtrHelper<IColumn, ColumnTuple>;
static Ptr create(const Columns & columns);
template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
std::string getName() const override;
const char * getFamilyName() const override { return "Tuple"; }
@ -49,9 +58,9 @@ public:
const char * deserializeAndInsertFromArena(const char * pos) override;
void updateHashWithValue(size_t n, SipHash & hash) const override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
MutableColumnPtr replicate(const Offsets & offsets) const override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr replicate(const Offsets & offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
void gather(ColumnGathererStream & gatherer_stream) override;
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

6
dbms/src/Columns/ColumnVector.cpp
View File

@ -146,7 +146,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
}
template <typename T>
MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
{
size_t size = data.size();
if (size != filt.size())
@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t r
}
template <typename T>
MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
{
size_t size = data.size();
@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, siz
}
template <typename T>
MutableColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
ColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
{
size_t size = data.size();
if (size != offsets.size())

6
dbms/src/Columns/ColumnVector.h
View File

@ -243,11 +243,11 @@ public:
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override;
ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
void getExtremes(Field & min, Field & max) const override;

3
dbms/src/Columns/FilterDescription.cpp
View File

@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column)
if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(&column))
{
MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate();
ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate();
ColumnUInt8 * concrete_column = typeid_cast<ColumnUInt8 *>(mutable_holder.get());
if (!concrete_column)

28
dbms/src/Columns/IColumn.h
View File

@ -45,7 +45,7 @@ public:
/** If column isn't constant, returns nullptr (or itself).
* If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
*/
virtual MutablePtr convertToFullColumnIfConst() const { return {}; }
virtual Ptr convertToFullColumnIfConst() const { return {}; }
/// Creates empty column with the same type.
virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
@ -104,11 +104,11 @@ public:
/// Removes all elements outside of specified range.
/// Is used in LIMIT operation, for example.
virtual MutablePtr cut(size_t start, size_t length) const
virtual Ptr cut(size_t start, size_t length) const
{
MutablePtr res = cloneEmpty();
res->insertRangeFrom(*this, start, length);
return res;
return std::move(res);
}
/// Appends new value at the end of column (column's size is increased by 1).
@ -171,12 +171,12 @@ public:
* otherwise (i.e. < 0), makes reserve() using size of source column.
*/
using Filter = PaddedPODArray<UInt8>;
virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
/// Permutes elements using specified permutation. Is used in sortings.
/// limit - if it isn't 0, puts only first limit elements in the result.
using Permutation = PaddedPODArray<size_t>;
virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0;
virtual Ptr permute(const Permutation & perm, size_t limit) const = 0;
/** Compares (*this)[n] and rhs[m].
* Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively.
@ -205,7 +205,7 @@ public:
*/
using Offset = UInt64;
using Offsets = PaddedPODArray<Offset>;
virtual MutablePtr replicate(const Offsets & offsets) const = 0;
virtual Ptr replicate(const Offsets & offsets) const = 0;
/** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
* Selector must contain values from 0 to num_columns - 1.
@ -247,10 +247,10 @@ public:
virtual void forEachSubcolumn(ColumnCallback) {}
MutablePtr mutate() const
MutablePtr mutate() const &&
{
MutablePtr res = COWPtr<IColumn>::mutate();
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); });
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); });
return res;
}
@ -360,4 +360,16 @@ using MutableColumns = std::vector<MutableColumnPtr>;
using ColumnRawPtrs = std::vector<const IColumn *>;
//using MutableColumnRawPtrs = std::vector<IColumn *>;
template <typename ... Args>
struct IsMutableColumns;
template <typename Arg, typename ... Args>
struct IsMutableColumns<Arg, Args ...>
{
static const bool value = std::is_assignable<MutableColumnPtr &&, Arg>::value && IsMutableColumns<Args ...>::value;
};
template <>
struct IsMutableColumns<> { static const bool value = true; };
}

6
dbms/src/Columns/IColumnDummy.h
View File

@ -74,12 +74,12 @@ public:
s += length;
}
MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
{
return cloneDummy(countBytesInFilter(filt));
}
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override
ColumnPtr permute(const Permutation & perm, size_t limit) const override
{
if (s != perm.size())
throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
@ -94,7 +94,7 @@ public:
res[i] = i;
}
MutableColumnPtr replicate(const Offsets & offsets) const override
ColumnPtr replicate(const Offsets & offsets) const override
{
if (s != offsets.size())
throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);

2
dbms/src/Common/AIO.h
View File

@ -1,6 +1,6 @@
#pragma once
#if !(defined(__FreeBSD__) || defined(__APPLE__))
#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
#include <Common/Exception.h>
#include <common/logger_useful.h>

23
dbms/src/Common/COWPtr.h
View File

@ -68,10 +68,9 @@
* of this shared state.
*
* Caveats:
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere
* and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified.
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere.
* - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better
* to use std::unique_ptr for it, but see above.
* to use std::unique_ptr for it somehow.
*/
template <typename Derived>
class COWPtr : public boost::intrusive_ref_counter<Derived>
@ -80,12 +79,22 @@ private:
Derived * derived() { return static_cast<Derived *>(this); }
const Derived * derived() const { return static_cast<const Derived *>(this); }
template <typename T>
class IntrusivePtr : public boost::intrusive_ptr<T>
{
public:
using boost::intrusive_ptr<T>::intrusive_ptr;
T & operator*() const & { return boost::intrusive_ptr<T>::operator*(); }
T && operator*() const && { return const_cast<typename std::remove_const<T>::type &&>(*boost::intrusive_ptr<T>::get()); }
};
protected:
template <typename T>
class mutable_ptr : public boost::intrusive_ptr<T>
class mutable_ptr : public IntrusivePtr<T>
{
private:
using Base = boost::intrusive_ptr<T>;
using Base = IntrusivePtr<T>;
template <typename> friend class COWPtr;
template <typename, typename> friend class COWPtrHelper;
@ -114,10 +123,10 @@ public:
protected:
template <typename T>
class immutable_ptr : public boost::intrusive_ptr<const T>
class immutable_ptr : public IntrusivePtr<const T>
{
private:
using Base = boost::intrusive_ptr<const T>;
using Base = IntrusivePtr<const T>;
template <typename> friend class COWPtr;
template <typename, typename> friend class COWPtrHelper;

28
dbms/src/Common/Config/ConfigProcessor.cpp
View File

@ -11,6 +11,7 @@
#include <Poco/DOM/Comment.h>
#include <Poco/Util/XMLConfiguration.h>
#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/StringUtils/StringUtils.h>
#define PREPROCESSED_SUFFIX "-preprocessed"
@ -276,6 +277,8 @@ void ConfigProcessor::doIncludesRecursive(
/// Replace the original contents, not add to it.
bool replace = attributes->getNamedItem("replace");
bool included_something = false;
auto process_include = [&](const Node * include_attr, const std::function<const Node * (const std::string &)> & get_node, const char * error_msg)
{
std::string name = include_attr->getNodeValue();
@ -316,6 +319,8 @@ void ConfigProcessor::doIncludesRecursive(
{
element->setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
}
included_something = true;
}
};
@ -348,11 +353,14 @@ void ConfigProcessor::doIncludesRecursive(
}
}
NodeListPtr children = node->childNodes();
Node * child = nullptr;
for (size_t i = 0; (child = children->item(i)); ++i)
if (included_something)
doIncludesRecursive(config, include_from, node, zk_node_cache, contributing_zk_paths);
else
{
doIncludesRecursive(config, include_from, child, zk_node_cache, contributing_zk_paths);
NodeListPtr children = node->childNodes();
Node * child = nullptr;
for (size_t i = 0; (child = children->item(i)); ++i)
doIncludesRecursive(config, include_from, child, zk_node_cache, contributing_zk_paths);
}
}
@ -375,15 +383,17 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
Poco::File merge_dir(merge_dir_name);
if (!merge_dir.exists() || !merge_dir.isDirectory())
continue;
for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it)
{
Poco::File & file = *it;
if (file.isFile()
&& (endsWith(file.path(), ".xml") || endsWith(file.path(), ".conf"))
&& !startsWith(file.path(), ".")) // skip temporary files
{
Poco::Path path(file.path());
std::string extension = path.getExtension();
std::string base_name = path.getBaseName();
// Skip non-config and temporary files
if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, "."))
files.push_back(file.path());
}
}
}

1
dbms/src/Common/Config/ConfigReloader.cpp
View File

@ -67,6 +67,7 @@ void ConfigReloader::run()
catch (...)
{
tryLogCurrentException(log, __PRETTY_FUNCTION__);
std::this_thread::sleep_for(reload_interval);
}
}
}

2
dbms/src/Common/CurrentMetrics.cpp
View File

@ -28,7 +28,9 @@
M(MemoryTrackingForMerges) \
M(LeaderElection) \
M(EphemeralNode) \
M(ZooKeeperSession) \
M(ZooKeeperWatch) \
M(ZooKeeperRequest) \
M(DelayedInserts) \
M(ContextLockWait) \
M(StorageBufferRows) \

1
dbms/src/Common/ErrorCodes.cpp
View File

@ -125,7 +125,6 @@ namespace ErrorCodes
extern const int UNKNOWN_SETTING = 115;
extern const int THERE_IS_NO_DEFAULT_VALUE = 116;
extern const int INCORRECT_DATA = 117;
extern const int TABLE_METADATA_DOESNT_EXIST = 118;
extern const int ENGINE_REQUIRED = 119;
extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE = 120;
extern const int UNKNOWN_SET_DATA_VARIANT = 121;

2
dbms/src/Common/Exception.cpp
View File

@ -11,7 +11,7 @@
#include <IO/ReadBufferFromString.h>
#include <Common/Exception.h>
#include <Common/demangle.h>
#include <common/demangle.h>
namespace DB

2
dbms/src/Common/FieldVisitors.h
View File

@ -3,7 +3,7 @@
#include <Core/Field.h>
#include <Core/AccurateComparison.h>
#include <common/DateLUT.h>
#include <Common/demangle.h>
#include <common/demangle.h>
class SipHash;

8
dbms/src/Common/ProfileEvents.cpp
View File

@ -56,14 +56,20 @@
\
M(ZooKeeperInit) \
M(ZooKeeperTransactions) \
M(ZooKeeperGetChildren) \
M(ZooKeeperList) \
M(ZooKeeperCreate) \
M(ZooKeeperRemove) \
M(ZooKeeperExists) \
M(ZooKeeperGet) \
M(ZooKeeperSet) \
M(ZooKeeperMulti) \
M(ZooKeeperCheck) \
M(ZooKeeperClose) \
M(ZooKeeperWatchResponse) \
M(ZooKeeperExceptions) \
M(ZooKeeperWaitMicroseconds) \
M(ZooKeeperBytesSent) \
M(ZooKeeperBytesReceived) \
\
M(DistributedConnectionFailTry) \
M(DistributedConnectionMissingTable) \

3
dbms/src/Common/ShellCommand.cpp
View File

@ -1,13 +1,12 @@
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <fcntl.h>
#include <dlfcn.h>
#include <Common/Exception.h>
#include <Common/ShellCommand.h>
#include <IO/WriteBufferFromVector.h>
#include <IO/WriteHelpers.h>
#include <port/unistd.h>
namespace DB

2
dbms/src/Common/StackTrace.cpp
View File

@ -7,7 +7,7 @@
#include <sstream>
#include <Common/StackTrace.h>
#include <Common/demangle.h>
#include <common/demangle.h>
StackTrace::StackTrace()

9
dbms/src/Common/Throttler.h
View File

@ -27,6 +27,9 @@ namespace ErrorCodes
class Throttler
{
public:
Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent = nullptr)
: max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent) {}
Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
const std::shared_ptr<Throttler> & parent = nullptr)
: max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent) {}
@ -76,6 +79,12 @@ public:
parent->add(amount);
}
/// Not thread safe
void setParent(const std::shared_ptr<Throttler> & parent_)
{
parent = parent_;
}
void reset()
{
std::lock_guard lock(mutex);

27
dbms/src/Common/UTF8Helpers.h
View File

@ -3,6 +3,9 @@
#include <Core/Types.h>
#include <Common/BitHelpers.h>
#if __SSE2__
#include <emmintrin.h>
#endif
namespace DB
{
@ -49,9 +52,29 @@ inline size_t seqLength(const UInt8 first_octet)
inline size_t countCodePoints(const UInt8 * data, size_t size)
{
size_t res = 0;
const auto end = data + size;
/// TODO SIMD implementation looks quite simple.
for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes.
#if __SSE2__
const auto bytes_sse = sizeof(__m128i);
const auto src_end_sse = (data + size) - (size % bytes_sse);
const auto align_sse = _mm_set1_epi8(0x40);
const auto upper_bound = _mm_set1_epi8(0xBF);
for (; data < src_end_sse; data += bytes_sse)
{
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
///Align to zero for the solve two case
const auto align_res = _mm_adds_epu8(chars, align_sse);
const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals));
}
#endif
for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
res += (*data <= 0x7F || *data >= 0xC0);
return res;

2
dbms/src/Common/Visitor.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Common/demangle.h>
#include <common/demangle.h>
#include <Common/TypeList.h>
#include <Common/Exception.h>

1
dbms/src/Common/ZooKeeper/CMakeLists.txt
View File

@ -1,4 +1,3 @@
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_common_zookeeper .)

4
dbms/src/Common/ZooKeeper/Increment.h
View File

@ -29,11 +29,11 @@ public:
if (zookeeper->tryGet(path, result_str, &stat))
{
result = std::stol(result_str) + 1;
success = zookeeper->trySet(path, std::to_string(result), stat.version) == ZOK;
success = zookeeper->trySet(path, std::to_string(result), stat.version) == ZooKeeperImpl::ZooKeeper::ZOK;
}
else
{
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == ZOK;
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == ZooKeeperImpl::ZooKeeper::ZOK;
}
}
while (!success);

113
dbms/src/Common/ZooKeeper/KeeperException.h
View File

@ -1,21 +1,6 @@
#pragma once
#include <Common/Exception.h>
#include "Types.h"
#include <Common/ProfileEvents.h>
namespace DB
{
namespace ErrorCodes
{
extern const int KEEPER_EXCEPTION;
}
}
namespace ProfileEvents
{
extern const Event ZooKeeperExceptions;
}
namespace zkutil
@ -23,98 +8,46 @@ namespace zkutil
/// You should reinitialize ZooKeeper session in case of these errors
inline bool isUnrecoverableErrorCode(int32_t zk_return_code)
inline bool isHardwareError(int32_t zk_return_code)
{
return zk_return_code == ZINVALIDSTATE || zk_return_code == ZSESSIONEXPIRED || zk_return_code == ZSESSIONMOVED;
}
/// Errors related with temporary network problems
inline bool isTemporaryErrorCode(int32_t zk_return_code)
{
return zk_return_code == ZCONNECTIONLOSS || zk_return_code == ZOPERATIONTIMEOUT;
}
/// Any error related with network or master election
/// In case of these errors you should retry the query or reinitialize ZooKeeper session (see isUnrecoverable())
inline bool isHardwareErrorCode(int32_t zk_return_code)
{
return isUnrecoverableErrorCode(zk_return_code) || isTemporaryErrorCode(zk_return_code);
return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT;
}
/// Valid errors sent from server
inline bool isUserError(int32_t zk_return_code)
{
return zk_return_code == ZNONODE
|| zk_return_code == ZBADVERSION
|| zk_return_code == ZNOCHILDRENFOREPHEMERALS
|| zk_return_code == ZNODEEXISTS
|| zk_return_code == ZNOTEMPTY;
return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY;
}
class KeeperException : public DB::Exception
{
private:
/// delegate constructor, used to minimize repetition; last parameter used for overload resolution
KeeperException(const std::string & msg, const int32_t code, int)
: DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code) { incrementEventCounter(); }
public:
explicit KeeperException(const std::string & msg) : KeeperException(msg, ZOK, 0) {}
KeeperException(const std::string & msg, const int32_t code)
: KeeperException(msg + " (" + zerror(code) + ")", code, 0) {}
explicit KeeperException(const int32_t code) : KeeperException(zerror(code), code, 0) {}
KeeperException(const int32_t code, const std::string & path)
: KeeperException(std::string{zerror(code)} + ", path: " + path, code, 0) {}
KeeperException(const KeeperException & exc) : DB::Exception(exc), code(exc.code) { incrementEventCounter(); }
const char * name() const throw() override { return "zkutil::KeeperException"; }
const char * className() const throw() override { return "zkutil::KeeperException"; }
KeeperException * clone() const override { return new KeeperException(*this); }
/// You should reinitialize ZooKeeper session in case of these errors
bool isUnrecoverable() const
{
return isUnrecoverableErrorCode(code);
}
/// Errors related with temporary network problems
bool isTemporaryError() const
{
return isTemporaryErrorCode(code);
}
/// Any error related with network or master election
/// In case of these errors you should retry the query or reinitialize ZooKeeper session (see isUnrecoverable())
bool isHardwareError() const
{
return isHardwareErrorCode(code);
}
const int32_t code;
private:
static void incrementEventCounter()
{
ProfileEvents::increment(ProfileEvents::ZooKeeperExceptions);
}
};
using KeeperException = ZooKeeperImpl::Exception;
class KeeperMultiException : public KeeperException
{
public:
MultiTransactionInfo info;
Requests requests;
Responses responses;
size_t failed_op_index = 0;
std::string getPathForFirstFailedOp() const;
/// If it is user error throws KeeperMultiException else throws ordinary KeeperException
/// If it is ZOK does nothing
static void check(const MultiTransactionInfo & info);
static void check(int code, const Ops & ops, const OpResultsPtr & op_results);
static void check(int32_t code, const Requests & requests, const Responses & responses);
protected:
KeeperMultiException(const MultiTransactionInfo & info, size_t failed_op_index);
KeeperMultiException(int32_t code, const Requests & requests, const Responses & responses);
private:
size_t getFailedOpIndex(int32_t code, const Responses & responses) const;
};
};

57
dbms/src/Common/ZooKeeper/LeaderElection.h
View File

@ -33,8 +33,7 @@ public:
*
* identifier - if not empty, must uniquely (within same path) identify participant of leader election.
* It means that different participants of leader election have different identifiers
* and existence of more than one ephemeral node with same identifier indicates an error
* (see cleanOldEphemeralNodes).
* and existence of more than one ephemeral node with same identifier indicates an error.
*/
LeaderElection(const std::string & path_, ZooKeeper & zookeeper_, LeadershipHandler handler_, const std::string & identifier_ = "")
: path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_)
@ -42,10 +41,15 @@ public:
createNode();
}
void yield()
void shutdown()
{
releaseNode();
createNode();
if (shutdown_called)
return;
shutdown_called = true;
event->set();
if (thread.joinable())
thread.join();
}
~LeaderElection()
@ -63,66 +67,31 @@ private:
std::string node_name;
std::thread thread;
std::atomic<bool> shutdown {false};
std::atomic<bool> shutdown_called {false};
zkutil::EventPtr event = std::make_shared<Poco::Event>();
CurrentMetrics::Increment metric_increment{CurrentMetrics::LeaderElection};
void createNode()
{
shutdown = false;
shutdown_called = false;
node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier);
std::string node_path = node->getPath();
node_name = node_path.substr(node_path.find_last_of('/') + 1);
cleanOldEphemeralNodes();
thread = std::thread(&LeaderElection::threadFunction, this);
}
void cleanOldEphemeralNodes()
{
if (identifier.empty())
return;
/** If there are nodes with same identifier, remove them.
* Such nodes could still be alive after failed attempt of removal,
* if it was temporary communication failure, that was continued for more than session timeout,
* but ZK session is still alive for unknown reason, and someone still holds that ZK session.
* See comments in destructor of EphemeralNodeHolder.
*/
Strings brothers = zookeeper.getChildren(path);
for (const auto & brother : brothers)
{
if (brother == node_name)
continue;
std::string brother_path = path + "/" + brother;
std::string brother_identifier = zookeeper.get(brother_path);
if (brother_identifier == identifier)
{
ProfileEvents::increment(ProfileEvents::ObsoleteEphemeralNode);
LOG_WARNING(&Logger::get("LeaderElection"), "Found obsolete ephemeral node for identifier "
+ identifier + ", removing: " + brother_path);
zookeeper.tryRemoveWithRetries(brother_path);
}
}
}
void releaseNode()
{
shutdown = true;
event->set();
if (thread.joinable())
thread.join();
shutdown();
node = nullptr;
}
void threadFunction()
{
while (!shutdown)
while (!shutdown_called)
{
bool success = false;

89
dbms/src/Common/ZooKeeper/Lock.cpp
View File

@ -1,5 +1,7 @@
#include "KeeperException.h"
#include "Lock.h"
using namespace zkutil;
bool Lock::tryLock()
@ -11,31 +13,16 @@ bool Lock::tryLock()
if (tryCheck() != Status::LOCKED_BY_ME)
locked.reset(nullptr);
}
if (!locked)
else
{
size_t attempt;
std::string dummy;
int32_t code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy);
/// TODO: ошибка. можно создать эфемерную ноду, но при этом не получить подтверждения даже после нескольких попыток.
/// тогда все последующие локи будут неуспешные из-за существования ноды.
int32_t code = zookeeper->tryCreateWithRetries(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy, &attempt);
if (code == ZNODEEXISTS)
if (code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS)
{
if (attempt == 0)
locked.reset(nullptr);
else
{
zkutil::Stat stat;
zookeeper->get(lock_path, &stat);
if (stat.ephemeralOwner == zookeeper->getClientID())
locked.reset(new ZooKeeperHandler(zookeeper));
else
locked.reset(nullptr);
}
locked.reset(nullptr);
}
else if (code == ZOK)
else if (code == ZooKeeperImpl::ZooKeeper::ZOK)
{
locked.reset(new ZooKeeperHandler(zookeeper));
}
@ -52,34 +39,8 @@ void Lock::unlock()
if (locked)
{
auto zookeeper = zookeeper_holder->getZooKeeper();
try
{
if (tryCheck() == Status::LOCKED_BY_ME)
{
size_t attempt;
int32_t code = zookeeper->tryRemoveEphemeralNodeWithRetries(lock_path, -1, &attempt);
if (attempt)
{
if (code != ZOK)
throw zkutil::KeeperException(code);
}
else
{
if (code == ZNONODE)
LOG_ERROR(log, "Node " << lock_path << " has been already removed. Probably due to network error.");
else if (code != ZOK)
throw zkutil::KeeperException(code);
}
}
}
catch (const zkutil::KeeperException & e)
{
/// если сессия находится в невостанавливаемом состоянии, то эфемерные ноды нам больше не принадлежат
/// и лок через таймаут будет отпущен
if (!e.isUnrecoverable())
throw;
}
if (tryCheck() == Status::LOCKED_BY_ME)
zookeeper->remove(lock_path, -1);
locked.reset(nullptr);
}
}
@ -97,47 +58,17 @@ Lock::Status Lock::tryCheck() const
else
{
if (stat.ephemeralOwner == zookeeper->getClientID())
{
lock_status = LOCKED_BY_ME;
}
else
{
lock_status = LOCKED_BY_OTHER;
}
}
if (locked && lock_status != LOCKED_BY_ME)
LOG_WARNING(log, "Lock is lost. It is normal if session was reinitialized. Path: " << lock_path << "/" << lock_message);
LOG_WARNING(log, "Lock is lost. It is normal if session was expired. Path: " << lock_path << "/" << lock_message);
return lock_status;
}
std::string Lock::status2String(Status status)
{
if (status >= END)
throw zkutil::KeeperException("Wrong status code: " + std::to_string(status));
static const char * names[] = {"Unlocked", "Locked by me", "Locked by other"};
return names[status];
}
void Lock::unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks)
{
try
{
unlock();
}
catch (const zkutil::KeeperException & e)
{
if (e.isTemporaryError())
{
LOG_WARNING(log, "Fail to unlock lock. Move lock to vector to remove later. Path: " << getPath());
failed_to_unlock_locks.emplace_back(std::move(*this));
}
else
throw;
}
}
void Lock::unlockAssumeLockNodeRemovedManually()
{
locked.reset(nullptr);

5
dbms/src/Common/ZooKeeper/Lock.h
View File

@ -40,7 +40,7 @@ namespace zkutil
{
unlock();
}
catch (const zkutil::KeeperException & e)
catch (...)
{
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
}
@ -51,15 +51,12 @@ namespace zkutil
UNLOCKED,
LOCKED_BY_ME,
LOCKED_BY_OTHER,
END
};
std::string status2String(Status status);
/// проверяет создана ли эфемерная нода и кто ее владелец.
Status tryCheck() const;
void unlock();
void unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks);
void unlockAssumeLockNodeRemovedManually();
bool tryLock();

213
dbms/src/Common/ZooKeeper/Types.h
View File

@ -3,184 +3,17 @@
#include <future>
#include <memory>
#include <vector>
#include <zookeeper.h>
#include <Common/ZooKeeper/ZooKeeperImpl.h>
#include <Poco/Event.h>
namespace zkutil
{
using ACLPtr = const ACL_vector *;
using Stat = ::Stat;
class ZooKeeper;
struct Op
{
public:
Op() : data(new zoo_op_t) {}
virtual ~Op() {}
virtual std::shared_ptr<Op> clone() const = 0;
virtual std::string getPath() const = 0;
virtual std::string describe() const = 0;
std::unique_ptr<zoo_op_t> data;
struct Remove;
struct Create;
struct SetData;
struct Check;
};
using OpPtr = std::shared_ptr<Op>;
struct Op::Remove : public Op
{
Remove(const std::string & path_, int32_t version_) :
path(path_), version(version_)
{
zoo_delete_op_init(data.get(), path.c_str(), version);
}
OpPtr clone() const override
{
return std::make_shared<Remove>(path, version);
}
std::string getPath() const override { return path; }
std::string describe() const override { return "command: remove, path: " + path; }
private:
std::string path;
int32_t version;
};
struct Op::Create : public Op
{
Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_);
OpPtr clone() const override
{
return std::make_shared<Create>(path_pattern, value, acl, flags);
}
std::string getPathCreated() { return created_path.data(); }
std::string getPath() const override { return path_pattern; }
std::string describe() const override
{
return "command: create"
", path: " + path_pattern +
", value: " + value;
}
private:
std::string path_pattern;
std::string value;
ACLPtr acl;
int32_t flags;
std::vector<char> created_path;
};
struct Op::SetData : public Op
{
SetData(const std::string & path_, const std::string & value_, int32_t version_) :
path(path_), value(value_), version(version_)
{
zoo_set_op_init(data.get(), path.c_str(), value.c_str(), value.size(), version, &stat);
}
OpPtr clone() const override
{
return std::make_shared<SetData>(path, value, version);
}
std::string getPath() const override { return path; }
std::string describe() const override
{
return
"command: set"
", path: " + path +
", value: " + value +
", version: " + std::to_string(data->set_op.version);
}
private:
std::string path;
std::string value;
int32_t version;
Stat stat;
};
struct Op::Check : public Op
{
Check(const std::string & path_, int32_t version_) :
path(path_), version(version_)
{
zoo_check_op_init(data.get(), path.c_str(), version);
}
OpPtr clone() const override
{
return std::make_shared<Check>(path, version);
}
std::string getPath() const override { return path; }
std::string describe() const override { return "command: check, path: " + path; }
private:
std::string path;
int32_t version;
};
using Ops = std::vector<OpPtr>;
/// C++ version of zoo_op_result_t
struct OpResult
{
int err;
std::string value;
std::unique_ptr<Stat> stat;
/// ZooKeeper is required for correct chroot path prefixes handling
explicit OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper = nullptr);
};
using OpResults = std::vector<OpResult>;
using OpResultsPtr = std::shared_ptr<OpResults>;
using Stat = ZooKeeperImpl::ZooKeeper::Stat;
using Strings = std::vector<std::string>;
/// Simple structure to handle transaction execution results
struct MultiTransactionInfo
{
Ops ops;
int32_t code = ZOK;
OpResultsPtr op_results;
MultiTransactionInfo() = default;
MultiTransactionInfo(int32_t code_, const Ops & ops_, const OpResultsPtr & op_results_)
: ops(ops_), code(code_), op_results(op_results_) {}
bool empty() const
{
return ops.empty();
}
/// Returns failed op if zkutil::isUserError(code) is true
const Op & getFailedOp() const;
};
namespace CreateMode
{
extern const int Persistent;
@ -191,23 +24,39 @@ namespace CreateMode
using EventPtr = std::shared_ptr<Poco::Event>;
class ZooKeeper;
/// Callback to call when the watch fires.
/// Because callbacks are called in the single "completion" thread internal to libzookeeper,
/// they must execute as quickly as possible (preferably just set some notification).
/// Parameters:
/// zookeeper - zookeeper session to which the fired watch belongs
/// type - event type, one of the *_EVENT constants from zookeeper.h
/// state - session connection state, one of the *_STATE constants from zookeeper.h
/// path - znode path to which the change happened. if event == ZOO_SESSION_EVENT it is either NULL or empty string.
using WatchCallback = std::function<void(ZooKeeper & zookeeper, int type, int state, const char * path)>;
using WatchCallback = ZooKeeperImpl::ZooKeeper::WatchCallback;
using Request = ZooKeeperImpl::ZooKeeper::Request;
using Response = ZooKeeperImpl::ZooKeeper::Response;
/// Returns first op which code != ZOK or throws an exception
/// ZooKeeper client sets correct OP codes if the transaction fails because of logical (user) errors like ZNODEEXISTS
/// If it is failed because of network error, for example, OP codes is not set.
/// Therefore you should make zkutil::isUserError() check before the function invocation.
size_t getFailedOpIndex(const OpResultsPtr & op_results, int32_t transaction_return_code);
using RequestPtr = ZooKeeperImpl::ZooKeeper::RequestPtr;
using ResponsePtr = ZooKeeperImpl::ZooKeeper::ResponsePtr;
using Requests = ZooKeeperImpl::ZooKeeper::Requests;
using Responses = ZooKeeperImpl::ZooKeeper::Responses;
using CreateRequest = ZooKeeperImpl::ZooKeeper::CreateRequest;
using RemoveRequest = ZooKeeperImpl::ZooKeeper::RemoveRequest;
using ExistsRequest = ZooKeeperImpl::ZooKeeper::ExistsRequest;
using GetRequest = ZooKeeperImpl::ZooKeeper::GetRequest;
using SetRequest = ZooKeeperImpl::ZooKeeper::SetRequest;
using ListRequest = ZooKeeperImpl::ZooKeeper::ListRequest;
using CheckRequest = ZooKeeperImpl::ZooKeeper::CheckRequest;
using CreateResponse = ZooKeeperImpl::ZooKeeper::CreateResponse;
using RemoveResponse = ZooKeeperImpl::ZooKeeper::RemoveResponse;
using ExistsResponse = ZooKeeperImpl::ZooKeeper::ExistsResponse;
using GetResponse = ZooKeeperImpl::ZooKeeper::GetResponse;
using SetResponse = ZooKeeperImpl::ZooKeeper::SetResponse;
using ListResponse = ZooKeeperImpl::ZooKeeper::ListResponse;
using CheckResponse = ZooKeeperImpl::ZooKeeper::CheckResponse;
RequestPtr makeCreateRequest(const std::string & path, const std::string & data, int create_mode);
RequestPtr makeRemoveRequest(const std::string & path, int version);
RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version);
RequestPtr makeCheckRequest(const std::string & path, int version);
}

1117
dbms/src/Common/ZooKeeper/ZooKeeper.cpp
View File

File diff suppressed because it is too large Load Diff

251
dbms/src/Common/ZooKeeper/ZooKeeper.h
View File

@ -1,17 +1,16 @@
#pragma once
#include "Types.h"
#include "KeeperException.h"
#include <Poco/Util/LayeredConfiguration.h>
#include <unordered_set>
#include <future>
#include <memory>
#include <mutex>
#include <string>
#include <unistd.h>
#include <common/logger_useful.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <port/unistd.h>
namespace ProfileEvents
@ -29,14 +28,10 @@ namespace zkutil
{
const UInt32 DEFAULT_SESSION_TIMEOUT = 30000;
const UInt32 MEDIUM_SESSION_TIMEOUT = 120000;
const UInt32 BIG_SESSION_TIMEOUT = 600000;
/// Preferred size of multi() command (in number of ops)
constexpr size_t MULTI_BATCH_SIZE = 100;
struct WatchContext;
struct MultiTransactionInfo;
/// ZooKeeper session. The interface is substantially different from the usual libzookeeper API.
///
@ -77,27 +72,15 @@ public:
*/
ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name);
~ZooKeeper();
/// Creates a new session with the same parameters. This method can be used for reconnecting
/// after the session has expired.
/// This object remains unchanged, and the new session is returned.
Ptr startNewSession() const;
/// Returns true, if the session has expired forever.
/// This is possible only if the connection has been established, then lost and re-established
/// again, but too late.
/// In contrast, if, for instance, the server name or port is misconfigured, connection
/// attempts will continue indefinitely, expired() will return false and all method calls
/// will raise ConnectionLoss exception.
/// Also returns true if is_dirty flag is set - a request to close the session ASAP.
/// Returns true, if the session has expired.
bool expired();
ACLPtr getDefaultACL();
void setDefaultACL(ACLPtr new_acl);
/// Create a znode. ACL set by setDefaultACL is used (full access to everybody by default).
/// Create a znode.
/// Throw an exception if something went wrong.
std::string create(const std::string & path, const std::string & data, int32_t mode);
@ -108,12 +91,9 @@ public:
/// In case of other errors throws an exception.
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode);
int32_t tryCreateWithRetries(const std::string & path, const std::string & data, int32_t mode,
std::string & path_created, size_t * attempt = nullptr);
/// Create a Persistent node.
/// Does nothing if the node already exists.
/// Retries on ConnectionLoss or OperationTimeout.
void createIfNotExists(const std::string & path, const std::string & data);
/// Creates all non-existent ancestors of the given path with empty contents.
@ -123,45 +103,14 @@ public:
/// Remove the node if the version matches. (if version == -1, remove any version).
void remove(const std::string & path, int32_t version = -1);
/// Removes the node. In case of network errors tries to remove again.
/// ZNONODE error for the second and the following tries is ignored.
void removeWithRetries(const std::string & path, int32_t version = -1);
/// Doesn't throw in the following cases:
/// * The node doesn't exist
/// * Versions don't match
/// * The node has children.
int32_t tryRemove(const std::string & path, int32_t version = -1);
/// Retries in case of network errors, returns ZNONODE if the node is already removed.
int32_t tryRemoveWithRetries(const std::string & path, int32_t version = -1, size_t * attempt = nullptr);
/// The same, but sets is_dirty flag if all removal attempts were unsuccessful.
/// This is needed because the session might still exist after all retries,
/// even if more time than session_timeout has passed.
/// So we do not rely on the ephemeral node being deleted and set is_dirty to
/// try and close the session ASAP.
/** Ridiculously Long Delay to Expire
When disconnects do happen, the common case should be a very* quick
reconnect to another server, but an extended network outage may
introduce a long delay before a client can reconnect to the ZooKeep‐
er service. Some developers wonder why the ZooKeeper client li‐
brary doesn’t simply decide at some point (perhaps twice the session
timeout) that enough is enough and kill the session itself.
There are two answers to this. First, ZooKeeper leaves this kind of
policy decision up to the developer. Developers can easily implement
such a policy by closing the handle themselves. Second, when a Zoo‐
Keeper ensemble goes down, time freezes. Thus, when the ensemble is
brought back up, session timeouts are restarted. If processes using
ZooKeeper hang in there, they may find out that the long timeout was
due to an extended ensemble failure that has recovered and pick right
up where they left off without any additional startup delay.
ZooKeeper: Distributed Process Coordination p118
*/
int32_t tryRemoveEphemeralNodeWithRetries(const std::string & path, int32_t version = -1, size_t * attempt = nullptr);
bool exists(const std::string & path, Stat * stat = nullptr, const EventPtr & watch = nullptr);
bool existsWatch(const std::string & path, Stat * stat, const WatchCallback & watch_callback);
bool existsWatch(const std::string & path, Stat * stat, WatchCallback watch_callback);
std::string get(const std::string & path, Stat * stat = nullptr, const EventPtr & watch = nullptr);
@ -169,7 +118,7 @@ public:
/// * The node doesn't exist. Returns false in this case.
bool tryGet(const std::string & path, std::string & res, Stat * stat = nullptr, const EventPtr & watch = nullptr, int * code = nullptr);
bool tryGetWatch(const std::string & path, std::string & res, Stat * stat, const WatchCallback & watch_callback, int * code = nullptr);
bool tryGetWatch(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback, int * code = nullptr);
void set(const std::string & path, const std::string & data,
int32_t version = -1, Stat * stat = nullptr);
@ -195,16 +144,14 @@ public:
/// Performs several operations in a transaction.
/// Throws on every error.
OpResultsPtr multi(const Ops & ops);
Responses multi(const Requests & requests);
/// Throws only if some operation has returned an "unexpected" error
/// - an error that would cause the corresponding try- method to throw.
int32_t tryMulti(const Ops & ops, OpResultsPtr * out_results = nullptr);
/// Use only with read-only operations.
int32_t tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results = nullptr, size_t * attempt = nullptr);
int32_t tryMulti(const Requests & requests, Responses & responses);
/// Throws nothing, just alias of multiImpl
int32_t tryMultiNoThrow(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr)
int32_t tryMultiNoThrow(const Requests & requests, Responses & responses)
{
return multiImpl(ops, out_op_results, out_info);
return multiImpl(requests, responses);
}
Int64 getClientID();
@ -237,182 +184,47 @@ public:
///
/// Future should not be destroyed before the result is gotten.
template <typename Result, typename... TaskParams>
class Future
{
friend class ZooKeeper;
private:
using Task = std::packaged_task<Result (TaskParams...)>;
using TaskPtr = std::unique_ptr<Task>;
using TaskPtrPtr = std::unique_ptr<TaskPtr>;
std::future<ZooKeeperImpl::ZooKeeper::GetResponse> asyncGet(const std::string & path);
/// Everything is complicated.
///
/// In libzookeeper async interface a function (e.g. zoo_aget)
/// accepts a pointer to a standalone callback function and void* pointer to the context
/// which is then passed to the callback.
/// The caller is responsible for ensuring that the context lives until the callback
/// is finished and we can't simply pass ownership of the context into function object.
/// Instead, we save the context in a Future object and return it to the caller.
/// The context will live until the Future lives.
/// Context data is wrapped in an unique_ptr so that its address (which is passed to
/// libzookeeper) remains unchanged after the Future is returned from the function.
///
/// The second problem is that after std::promise has been fulfilled, and the user
/// has gotten the result from std::future, the Future object can be destroyed
/// before the std::promise::set_value() call that fulfils the promise completes in another
/// thread.
/// See http://stackoverflow.com/questions/10843304/race-condition-in-pthread-once
/// To combat this we use the second unique_ptr. Inside the callback, the void* context
/// is cast to unique_ptr and moved into the local unique_ptr to prolong the lifetime of
/// the context data.
std::future<ZooKeeperImpl::ZooKeeper::GetResponse> asyncTryGet(const std::string & path);
TaskPtrPtr task;
std::future<Result> future;
std::future<ZooKeeperImpl::ZooKeeper::ExistsResponse> asyncExists(const std::string & path);
template <typename... Args>
Future(Args &&... args) :
task(std::make_unique<TaskPtr>(std::make_unique<Task>(std::forward<Args>(args)...))),
future((*task)->get_future()) {}
std::future<ZooKeeperImpl::ZooKeeper::ListResponse> asyncGetChildren(const std::string & path);
public:
Result get()
{
return future.get();
}
Future(Future &&) = default;
Future & operator= (Future &&) = default;
~Future()
{
/// If nobody has waited for the result, we must wait for it before the object is
/// destroyed, because the object contents can still be used in the callback.
if (future.valid())
future.wait();
}
};
struct ValueAndStat
{
std::string value;
Stat stat;
};
using GetFuture = Future<ValueAndStat, int, const char *, int, const Stat *>;
GetFuture asyncGet(const std::string & path);
struct ValueAndStatAndExists
{
std::string value;
Stat stat;
bool exists;
};
using TryGetFuture = Future<ValueAndStatAndExists, int, const char *, int, const Stat *>;
TryGetFuture asyncTryGet(const std::string & path);
struct StatAndExists
{
Stat stat;
bool exists;
};
using ExistsFuture = Future<StatAndExists, int, const Stat *>;
ExistsFuture asyncExists(const std::string & path);
using GetChildrenFuture = Future<Strings, int, const String_vector *>;
GetChildrenFuture asyncGetChildren(const std::string & path);
using RemoveFuture = Future<void, int>;
RemoveFuture asyncRemove(const std::string & path, int32_t version = -1);
std::future<ZooKeeperImpl::ZooKeeper::RemoveResponse> asyncRemove(const std::string & path, int32_t version = -1);
/// Doesn't throw in the following cases:
/// * The node doesn't exist
/// * The versions do not match
/// * The node has children
using TryRemoveFuture = Future<int32_t, int>;
TryRemoveFuture asyncTryRemove(const std::string & path, int32_t version = -1);
std::future<ZooKeeperImpl::ZooKeeper::RemoveResponse> asyncTryRemove(const std::string & path, int32_t version = -1);
struct OpResultsAndCode
{
OpResultsPtr results;
std::shared_ptr<Ops> ops_ptr;
int code;
};
std::future<ZooKeeperImpl::ZooKeeper::MultiResponse> asyncMulti(const Requests & ops);
using MultiFuture = Future<OpResultsAndCode, int>;
MultiFuture asyncMulti(const Ops & ops);
/// Like the previous one but don't throw any exceptions on future.get()
MultiFuture tryAsyncMulti(const Ops & ops);
std::future<ZooKeeperImpl::ZooKeeper::MultiResponse> tryAsyncMulti(const Requests & ops);
static std::string error2string(int32_t code);
/// Max size of node contents in bytes.
/// In 3.4.5 max node size is 1Mb.
static const size_t MAX_NODE_SIZE = 1048576;
/// Length of the suffix that ZooKeeper adds to sequential nodes.
/// In fact it is smaller, but round it up for convenience.
static const size_t SEQUENTIAL_SUFFIX_SIZE = 64;
zhandle_t * getHandle() { return impl; }
private:
friend struct WatchContext;
friend class EphemeralNodeHolder;
friend struct OpResult;
void init(const std::string & hosts, const std::string & identity, int32_t session_timeout_ms, const std::string & chroot);
void init(const std::string & hosts_, const std::string & identity_, int32_t session_timeout_ms_, const std::string & chroot_);
void removeChildrenRecursive(const std::string & path);
void tryRemoveChildrenRecursive(const std::string & path);
static WatchCallback callbackForEvent(const EventPtr & event);
WatchContext * createContext(WatchCallback && callback);
static void destroyContext(WatchContext * context);
static void processCallback(zhandle_t * zh, int type, int state, const char * path, void * watcher_ctx);
template <typename T>
int32_t retry(T && operation, size_t * attempt = nullptr)
{
int32_t code = operation();
if (attempt)
*attempt = 0;
for (size_t i = 0; (i < retry_num) && (code == ZOPERATIONTIMEOUT || code == ZCONNECTIONLOSS); ++i)
{
if (attempt)
*attempt = i;
/// If the connection has been lost, wait timeout/3 hoping for connection re-establishment.
static const int MAX_SLEEP_TIME = 10;
if (code == ZCONNECTIONLOSS)
usleep(std::min(session_timeout_ms * 1000u / 3, MAX_SLEEP_TIME * 1000u * 1000u));
LOG_WARNING(log, "Error on attempt " << i << ": " << error2string(code) << ". Retry");
code = operation();
}
return code;
}
/// The following methods don't throw exceptions but return error codes.
int32_t createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
int32_t removeImpl(const std::string & path, int32_t version = -1);
int32_t removeImpl(const std::string & path, int32_t version);
int32_t getImpl(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback);
int32_t setImpl(const std::string & path, const std::string & data, int32_t version = -1, Stat * stat = nullptr);
int32_t setImpl(const std::string & path, const std::string & data, int32_t version, Stat * stat);
int32_t getChildrenImpl(const std::string & path, Strings & res, Stat * stat, WatchCallback watch_callback);
int32_t multiImpl(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr);
int32_t multiImpl(const Requests & requests, Responses & responses);
int32_t existsImpl(const std::string & path, Stat * stat_, WatchCallback watch_callback);
MultiFuture asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception);
std::unique_ptr<ZooKeeperImpl::ZooKeeper> impl;
std::string hosts;
std::string identity;
@ -420,19 +232,8 @@ private:
std::string chroot;
std::mutex mutex;
ACLPtr default_acl;
zhandle_t * impl;
std::unordered_set<WatchContext *> watch_context_store;
/// Retries number in case of OperationTimeout or ConnectionLoss errors.
static constexpr size_t retry_num = 3;
Logger * log = nullptr;
/// If true, there were unsuccessfull attempts to remove ephemeral nodes.
/// It is better to close the session to remove ephemeral nodes with certainty
/// instead of continuing to use re-established session.
bool is_dirty = false;
};
@ -476,13 +277,9 @@ public:
{
try
{
/// Important: if the ZooKeeper is temporarily unavailable, repeated attempts to
/// delete the node are made.
/// Otherwise it is possible that EphemeralNodeHolder is destroyed,
/// but the session has recovered and the node in ZooKeeper remains for the long time.
zookeeper.tryRemoveEphemeralNodeWithRetries(path);
zookeeper.tryRemove(path);
}
catch (const KeeperException & e)
catch (...)
{
ProfileEvents::increment(ProfileEvents::CannotRemoveEphemeralNode);
DB::tryLogCurrentException(__PRETTY_FUNCTION__);

1541
dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

635
dbms/src/Common/ZooKeeper/ZooKeeperImpl.h Normal file
View File

@ -0,0 +1,635 @@
#pragma once
#include <Core/Types.h>
#include <Common/ConcurrentBoundedQueue.h>
#include <Common/CurrentMetrics.h>
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <IO/ReadBufferFromPocoSocket.h>
#include <IO/WriteBufferFromPocoSocket.h>
#include <Poco/Net/StreamSocket.h>
#include <Poco/Net/SocketAddress.h>
#include <map>
#include <mutex>
#include <chrono>
#include <vector>
#include <memory>
#include <thread>
#include <atomic>
#include <cstdint>
#include <optional>
#include <functional>
/** ZooKeeper C++ library, a replacement for libzookeeper.
*
* Motivation.
*
* libzookeeper has many bugs:
* - segfaults: for example, if zookeeper connection was interrupted while reading result of multi response;
* - memory corruption: for example, as a result of double free inside libzookeeper;
* - no timeouts for synchronous operations: they may stuck forever under simple Jepsen-like tests;
* - logical errors: for example, chroot prefix is not removed from the results of multi responses.
* - data races;
*
* The code of libzookeeper is over complicated:
* - memory ownership is unclear and bugs are very difficult to track and fix.
* - extremely creepy code for implementation of "chroot" feature.
*
* As of 2018, there are no active maintainers of libzookeeper:
* - bugs in JIRA are fixed only occasionaly with ad-hoc patches by library users.
*
* libzookeeper is a classical example of bad code written in C.
*
* In Go, Python and Rust programming languages,
* there are separate libraries for ZooKeeper, not based on libzookeeper.
* Motivation is almost the same. Example:
* https://github.com/python-zk/kazoo/blob/master/docs/implementation.rst
*
* About "session restore" feature.
*
* libzookeeper has the feature of session restore. Client receives session id and session token from the server,
* and when connection is lost, it can quickly reconnect to any server with the same session id and token,
* to continue with existing session.
* libzookeeper performs this reconnection automatically.
*
* This feature is proven to be harmful.
* For example, it makes very difficult to correctly remove ephemeral nodes.
* This may lead to weird bugs in application code.
* For example, our developers have found that type of bugs in Curator Java library.
*
* On the other side, session restore feature has no advantages,
* because every application should be able to establish new session and reinitialize internal state,
* when the session is lost and cannot be restored.
*
* This library never restores the session. In case of any error, the session is considered as expired
* and you should create a new instance of ZooKeeper object and reinitialize the application state.
*
* This library is not intended to be CPU efficient. Hundreds of thousands operations per second is usually enough.
*/
namespace CurrentMetrics
{
extern const Metric ZooKeeperSession;
}
namespace ZooKeeperImpl
{
using namespace DB;
class Exception : public DB::Exception
{
private:
/// Delegate constructor, used to minimize repetition; last parameter used for overload resolution.
Exception(const std::string & msg, const int32_t code, int);
public:
explicit Exception(const int32_t code);
Exception(const std::string & msg, const int32_t code);
Exception(const int32_t code, const std::string & path);
Exception(const Exception & exc);
const char * name() const throw() override { return "ZooKeeperImpl::Exception"; }
const char * className() const throw() override { return "ZooKeeperImpl::Exception"; }
Exception * clone() const override { return new Exception(*this); }
const int32_t code;
};
/** Usage scenario:
* - create an object and issue commands;
* - you provide callbacks for your commands; callbacks are invoked in internal thread and must be cheap:
* for example, just signal a condvar / fulfull a promise.
* - you also may provide callbacks for watches; they are also invoked in internal thread and must be cheap.
* - whenever you receive exception with ZSESSIONEXPIRED code or method isExpired returns true,
* the ZooKeeper instance is no longer usable - you may only destroy it and probably create another.
* - whenever session is expired or ZooKeeper instance is destroying, all callbacks are notified with special event.
* - data for callbacks must be alive when ZooKeeper instance is alive.
*/
class ZooKeeper
{
public:
using Addresses = std::vector<Poco::Net::SocketAddress>;
struct ACL
{
static constexpr int32_t Read = 1;
static constexpr int32_t Write = 2;
static constexpr int32_t Create = 4;
static constexpr int32_t Delete = 8;
static constexpr int32_t Admin = 16;
static constexpr int32_t All = 0x1F;
int32_t permissions;
String scheme;
String id;
void write(WriteBuffer & out) const;
};
using ACLs = std::vector<ACL>;
struct Stat
{
int64_t czxid;
int64_t mzxid;
int64_t ctime;
int64_t mtime;
int32_t version;
int32_t cversion;
int32_t aversion;
int64_t ephemeralOwner;
int32_t dataLength;
int32_t numChildren;
int64_t pzxid;
void read(ReadBuffer & in);
};
using XID = int32_t;
using OpNum = int32_t;
struct Response
{
int32_t error = 0;
virtual ~Response() {}
virtual void readImpl(ReadBuffer &) = 0;
virtual void removeRootPath(const String & /* root_path */) {}
};
using ResponsePtr = std::shared_ptr<Response>;
using Responses = std::vector<ResponsePtr>;
using ResponseCallback = std::function<void(const Response &)>;
struct Request
{
XID xid = 0;
bool has_watch = false;
virtual ~Request() {};
virtual OpNum getOpNum() const = 0;
/// Writes length, xid, op_num, then the rest.
void write(WriteBuffer & out) const;
virtual void writeImpl(WriteBuffer &) const = 0;
virtual ResponsePtr makeResponse() const = 0;
virtual void addRootPath(const String & /* root_path */) {};
virtual String getPath() const = 0;
};
using RequestPtr = std::shared_ptr<Request>;
using Requests = std::vector<RequestPtr>;
struct HeartbeatRequest final : Request
{
OpNum getOpNum() const override { return 11; }
void writeImpl(WriteBuffer &) const override {}
ResponsePtr makeResponse() const override;
String getPath() const override { return {}; }
};
struct HeartbeatResponse final : Response
{
void readImpl(ReadBuffer &) override {}
};
struct WatchResponse final : Response
{
int32_t type = 0;
int32_t state = 0;
String path;
void readImpl(ReadBuffer &) override;
void removeRootPath(const String & root_path) override;
};
using WatchCallback = std::function<void(const WatchResponse &)>;
struct AuthRequest final : Request
{
int32_t type = 0; /// ignored by the server
String scheme;
String data;
OpNum getOpNum() const override { return 100; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
String getPath() const override { return {}; }
};
struct AuthResponse final : Response
{
void readImpl(ReadBuffer &) override {};
};
struct CloseRequest final : Request
{
OpNum getOpNum() const override { return -11; }
void writeImpl(WriteBuffer &) const override {}
ResponsePtr makeResponse() const override;
String getPath() const override { return {}; }
};
struct CloseResponse final : Response
{
void readImpl(ReadBuffer &) override;
};
struct CreateRequest final : Request
{
String path;
String data;
bool is_ephemeral = false;
bool is_sequential = false;
ACLs acls;
OpNum getOpNum() const override { return 1; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct CreateResponse final : Response
{
String path_created;
void readImpl(ReadBuffer &) override;
void removeRootPath(const String & root_path) override;
};
struct RemoveRequest final : Request
{
String path;
int32_t version = -1;
OpNum getOpNum() const override { return 2; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct RemoveResponse final : Response
{
void readImpl(ReadBuffer &) override {}
};
struct ExistsRequest final : Request
{
String path;
OpNum getOpNum() const override { return 3; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct ExistsResponse final : Response
{
Stat stat;
void readImpl(ReadBuffer &) override;
};
struct GetRequest final : Request
{
String path;
OpNum getOpNum() const override { return 4; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct GetResponse final : Response
{
String data;
Stat stat;
void readImpl(ReadBuffer &) override;
};
struct SetRequest final : Request
{
String path;
String data;
int32_t version = -1;
OpNum getOpNum() const override { return 5; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct SetResponse final : Response
{
Stat stat;
void readImpl(ReadBuffer &) override;
};
struct ListRequest final : Request
{
String path;
OpNum getOpNum() const override { return 12; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct ListResponse final : Response
{
std::vector<String> names;
Stat stat;
void readImpl(ReadBuffer &) override;
};
struct CheckRequest final : Request
{
String path;
int32_t version = -1;
OpNum getOpNum() const override { return 13; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return path; }
};
struct CheckResponse final : Response
{
void readImpl(ReadBuffer &) override {};
};
struct MultiRequest final : Request
{
Requests requests;
OpNum getOpNum() const override { return 14; }
void writeImpl(WriteBuffer &) const override;
ResponsePtr makeResponse() const override;
void addRootPath(const String & root_path) override;
String getPath() const override { return {}; }
};
struct MultiResponse final : Response
{
Responses responses;
MultiResponse(const Requests & requests);
void readImpl(ReadBuffer &) override;
void removeRootPath(const String & root_path) override;
};
/// This response may be received only as an element of responses in MultiResponse.
struct ErrorResponse final : Response
{
void readImpl(ReadBuffer &) override;
};
/** Connection to addresses is performed in order. If you want, shuffle them manually.
* Operation timeout couldn't be greater than session timeout.
* Operation timeout applies independently for network read, network write, waiting for events and synchronization.
*/
ZooKeeper(
const Addresses & addresses,
const String & root_path,
const String & auth_scheme,
const String & auth_data,
Poco::Timespan session_timeout,
Poco::Timespan connection_timeout,
Poco::Timespan operation_timeout);
~ZooKeeper();
/// If expired, you can only destroy the object. All other methods will throw exception.
bool isExpired() const { return expired; }
/// Useful to check owner of ephemeral node.
int64_t getSessionID() const { return session_id; }
using CreateCallback = std::function<void(const CreateResponse &)>;
using RemoveCallback = std::function<void(const RemoveResponse &)>;
using ExistsCallback = std::function<void(const ExistsResponse &)>;
using GetCallback = std::function<void(const GetResponse &)>;
using SetCallback = std::function<void(const SetResponse &)>;
using ListCallback = std::function<void(const ListResponse &)>;
using CheckCallback = std::function<void(const CheckResponse &)>;
using MultiCallback = std::function<void(const MultiResponse &)>;
/// If the method will throw an exception, callbacks won't be called.
///
/// After the method is executed successfully, you must wait for callbacks
/// (don't destroy callback data before it will be called).
///
/// All callbacks are executed sequentially (the execution of callbacks is serialized).
///
/// If an exception is thrown inside the callback, the session will expire,
/// and all other callbacks will be called with "Session expired" error.
void create(
const String & path,
const String & data,
bool is_ephemeral,
bool is_sequential,
const ACLs & acls,
CreateCallback callback);
void remove(
const String & path,
int32_t version,
RemoveCallback callback);
void exists(
const String & path,
ExistsCallback callback,
WatchCallback watch);
void get(
const String & path,
GetCallback callback,
WatchCallback watch);
void set(
const String & path,
const String & data,
int32_t version,
SetCallback callback);
void list(
const String & path,
ListCallback callback,
WatchCallback watch);
void check(
const String & path,
int32_t version,
CheckCallback callback);
void multi(
const Requests & requests,
MultiCallback callback);
enum Error
{
ZOK = 0,
/** System and server-side errors.
* This is never thrown by the server, it shouldn't be used other than
* to indicate a range. Specifically error codes greater than this
* value, but lesser than ZAPIERROR, are system errors.
*/
ZSYSTEMERROR = -1,
ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found
ZDATAINCONSISTENCY = -3, /// A data inconsistency was found
ZCONNECTIONLOSS = -4, /// Connection to the server has been lost
ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data
ZUNIMPLEMENTED = -6, /// Operation is unimplemented
ZOPERATIONTIMEOUT = -7, /// Operation timeout
ZBADARGUMENTS = -8, /// Invalid arguments
ZINVALIDSTATE = -9, /// Invliad zhandle state
/** API errors.
* This is never thrown by the server, it shouldn't be used other than
* to indicate a range. Specifically error codes greater than this
* value are API errors.
*/
ZAPIERROR = -100,
ZNONODE = -101, /// Node does not exist
ZNOAUTH = -102, /// Not authenticated
ZBADVERSION = -103, /// Version conflict
ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children
ZNODEEXISTS = -110, /// The node already exists
ZNOTEMPTY = -111, /// The node has children
ZSESSIONEXPIRED = -112, /// The session has been expired by the server
ZINVALIDCALLBACK = -113, /// Invalid callback specified
ZINVALIDACL = -114, /// Invalid ACL specified
ZAUTHFAILED = -115, /// Client authentication failed
ZCLOSING = -116, /// ZooKeeper is closing
ZNOTHING = -117, /// (not error) no server responses to process
ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored
};
static const char * errorMessage(int32_t code);
/// For watches.
enum State
{
EXPIRED_SESSION = -112,
AUTH_FAILED = -113,
CONNECTING = 1,
ASSOCIATING = 2,
CONNECTED = 3,
NOTCONNECTED = 999
};
enum Event
{
CREATED = 1,
DELETED = 2,
CHANGED = 3,
CHILD = 4,
SESSION = -1,
NOTWATCHING = -2
};
private:
String root_path;
ACLs default_acls;
Poco::Timespan session_timeout;
Poco::Timespan operation_timeout;
Poco::Net::StreamSocket socket;
std::optional<ReadBufferFromPocoSocket> in;
std::optional<WriteBufferFromPocoSocket> out;
int64_t session_id = 0;
std::atomic<XID> xid {1};
std::atomic<bool> expired {false};
std::mutex finalize_mutex;
using clock = std::chrono::steady_clock;
struct RequestInfo
{
RequestPtr request;
ResponseCallback callback;
WatchCallback watch;
clock::time_point time;
};
using RequestsQueue = ConcurrentBoundedQueue<RequestInfo>;
RequestsQueue requests_queue{1};
void pushRequest(RequestInfo && request);
using Operations = std::map<XID, RequestInfo>;
Operations operations;
std::mutex operations_mutex;
using WatchCallbacks = std::vector<WatchCallback>;
using Watches = std::map<String /* path */, WatchCallbacks>;
Watches watches;
std::mutex watches_mutex;
std::thread send_thread;
std::thread receive_thread;
void connect(
const Addresses & addresses,
Poco::Timespan connection_timeout);
void sendHandshake();
void receiveHandshake();
void sendAuth(const String & scheme, const String & data);
void receiveEvent();
void sendThread();
void receiveThread();
void close();
/// Call all remaining callbacks and watches, passing errors to them.
void finalize(bool error_send, bool error_receive);
template <typename T>
void write(const T &);
template <typename T>
void read(T &);
CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession};
};
};

12
dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp
View File

@ -41,20 +41,18 @@ std::optional<std::string> ZooKeeperNodeCache::get(const std::string & path)
if (nonexistent_nodes.count(path))
return std::nullopt;
auto watch_callback = [context=context](zkutil::ZooKeeper & zookeeper, int type, int state, const char * path)
auto watch_callback = [context=context](const ZooKeeperImpl::ZooKeeper::WatchResponse & response)
{
if (!(type != ZOO_SESSION_EVENT || state == ZOO_EXPIRED_SESSION_STATE))
if (!(response.type != ZooKeeperImpl::ZooKeeper::SESSION || response.state == ZooKeeperImpl::ZooKeeper::EXPIRED_SESSION))
return;
bool changed = false;
{
std::lock_guard<std::mutex> lock(context->mutex);
if (&zookeeper != context->zookeeper.get())
return;
if (type != ZOO_SESSION_EVENT)
changed = context->invalidated_paths.emplace(path).second;
else if (state == ZOO_EXPIRED_SESSION_STATE)
if (response.type != ZooKeeperImpl::ZooKeeper::SESSION)
changed = context->invalidated_paths.emplace(response.path).second;
else if (response.state == ZooKeeperImpl::ZooKeeper::EXPIRED_SESSION)
{
context->zookeeper = nullptr;
context->invalidated_paths.clear();

6
dbms/src/Common/ZooKeeper/tests/CMakeLists.txt
View File

@ -1,6 +1,9 @@
add_executable(zkutil_test_commands zkutil_test_commands.cpp)
target_link_libraries(zkutil_test_commands clickhouse_common_zookeeper)
add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp)
target_link_libraries(zkutil_test_commands_new_lib clickhouse_common_zookeeper)
add_executable(zkutil_test_lock zkutil_test_lock.cpp)
target_link_libraries(zkutil_test_lock clickhouse_common_zookeeper)
@ -15,3 +18,6 @@ target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper)
add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp)
target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config)
add_executable (zookeeper_impl zookeeper_impl.cpp)
target_link_libraries (zookeeper_impl clickhouse_common_zookeeper)

73
dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
View File

@ -1,4 +1,6 @@
#include <Common/typeid_cast.h>
#include <Common/ZooKeeper/ZooKeeper.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/StringUtils/StringUtils.h>
#include <iostream>
#include <chrono>
@ -31,25 +33,24 @@ TEST(zkutil, multi_nice_exception_msg)
{
auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
auto acl = zookeeper->getDefaultACL();
zkutil::Ops ops;
zkutil::Requests ops;
ASSERT_NO_THROW(
zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent));
zookeeper->multi(ops);
);
try
{
ops.clear();
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/c", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi/c", -1));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "BadBoy", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/b", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/c", "_", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeRemoveRequest("/clickhouse_test/zkutil_multi/c", -1));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "BadBoy", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/b", "_", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent));
zookeeper->multi(ops);
FAIL();
@ -69,8 +70,7 @@ TEST(zkutil, multi_nice_exception_msg)
TEST(zkutil, multi_async)
{
auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
auto acl = zookeeper->getDefaultACL();
zkutil::Ops ops;
zkutil::Requests ops;
zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
@ -81,30 +81,29 @@ TEST(zkutil, multi_async)
{
ops.clear();
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "", zkutil::CreateMode::Persistent));
auto fut = zookeeper->tryAsyncMulti(ops);
ops.clear();
auto res = fut.get();
ASSERT_TRUE(res.code == ZOK);
ASSERT_EQ(res.results->size(), 2);
ASSERT_EQ(res.ops_ptr->size(), 2);
ASSERT_TRUE(res.error == ZooKeeperImpl::ZooKeeper::ZOK);
ASSERT_EQ(res.responses.size(), 2);
}
EXPECT_ANY_THROW
(
std::vector<zkutil::ZooKeeper::MultiFuture> futures;
std::vector<std::future<ZooKeeperImpl::ZooKeeper::MultiResponse>> futures;
for (size_t i = 0; i < 10000; ++i)
{
ops.clear();
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi", -1));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", -1));
ops.emplace_back(new zkutil::Op::SetData("/clickhouse_test/zkutil_multi", "xxx", 42));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeRemoveRequest("/clickhouse_test/zkutil_multi", -1));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCheckRequest("/clickhouse_test/zkutil_multi", -1));
ops.emplace_back(zkutil::makeSetRequest("/clickhouse_test/zkutil_multi", "xxx", 42));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent));
futures.emplace_back(zookeeper->asyncMulti(ops));
}
@ -118,16 +117,15 @@ TEST(zkutil, multi_async)
{
ops.clear();
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent));
auto fut = zookeeper->tryAsyncMulti(ops);
ops.clear();
auto res = fut.get();
ASSERT_TRUE(res.code == ZNODEEXISTS);
ASSERT_EQ(res.results->size(), 2);
ASSERT_EQ(res.ops_ptr->size(), 2);
ASSERT_TRUE(res.error == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS);
ASSERT_EQ(res.responses.size(), 2);
}
}
@ -135,9 +133,9 @@ TEST(zkutil, multi_async)
TEST(zkutil, multi_async_libzookeeper_segfault)
{
auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", 1000);
zkutil::Ops ops;
zkutil::Requests ops;
ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", 0));
ops.emplace_back(zkutil::makeCheckRequest("/clickhouse_test/zkutil_multi", 0));
/// Uncomment to test
//auto cmd = ShellCommand::execute("sudo service zookeeper restart");
@ -146,7 +144,7 @@ TEST(zkutil, multi_async_libzookeeper_segfault)
auto future = zookeeper->asyncMulti(ops);
auto res = future.get();
EXPECT_TRUE(zkutil::isUnrecoverableErrorCode(res.code));
EXPECT_TRUE(zkutil::isHardwareError(res.error));
}
@ -159,21 +157,20 @@ TEST(zkutil, multi_create_sequential)
zookeeper->createAncestors("/clickhouse_test/");
zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", zkutil::DEFAULT_SESSION_TIMEOUT, "/clickhouse_test");
auto acl = zookeeper->getDefaultACL();
zkutil::Ops ops;
zkutil::Requests ops;
String base_path = "/zkutil/multi_create_sequential";
zookeeper->tryRemoveRecursive(base_path);
zookeeper->createAncestors(base_path + "/");
String sequential_node_prefix = base_path + "/queue-";
ops.emplace_back(new zkutil::Op::Create(sequential_node_prefix, "", acl, zkutil::CreateMode::EphemeralSequential));
zkutil::OpResultsPtr results = zookeeper->multi(ops);
zkutil::OpResult & sequential_node_result_op = results->at(0);
ops.emplace_back(zkutil::makeCreateRequest(sequential_node_prefix, "", zkutil::CreateMode::EphemeralSequential));
auto results = zookeeper->multi(ops);
const auto & sequential_node_result_op = typeid_cast<const zkutil::CreateResponse &>(*results.at(0));
EXPECT_FALSE(sequential_node_result_op.value.empty());
EXPECT_GT(sequential_node_result_op.value.length(), sequential_node_prefix.length());
EXPECT_EQ(sequential_node_result_op.value.substr(0, sequential_node_prefix.length()), sequential_node_prefix);
EXPECT_FALSE(sequential_node_result_op.path_created.empty());
EXPECT_GT(sequential_node_result_op.path_created.length(), sequential_node_prefix.length());
EXPECT_EQ(sequential_node_result_op.path_created.substr(0, sequential_node_prefix.length()), sequential_node_prefix);
}
catch (...)
{

17
dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
View File

@ -1,5 +1,6 @@
#include <iostream>
#include <Common/ZooKeeper/ZooKeeper.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Poco/ConsoleChannel.h>
#include <Common/Exception.h>
@ -33,18 +34,18 @@ int main(int argc, char ** argv)
while (true)
{
{
zkutil::Ops ops;
ops.emplace_back(std::make_shared<zkutil::Op::Create>("/test/zk_expiration_test", "hello", zk.getDefaultACL(), zkutil::CreateMode::Persistent));
ops.emplace_back(std::make_shared<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
zkutil::Requests ops;
ops.emplace_back(zkutil::makeCreateRequest("/test/zk_expiration_test", "hello", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeRemoveRequest("/test/zk_expiration_test", -1));
zkutil::MultiTransactionInfo info;
zk.tryMultiNoThrow(ops, nullptr, &info);
zkutil::Responses responses;
int32_t code = zk.tryMultiNoThrow(ops, responses);
std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(info.code) << std::endl;
std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(code) << std::endl;
try
{
if (info.code != ZOK)
std::cout << "Path: " << info.getFailedOp().getPath() << std::endl;
if (code)
std::cout << "Path: " << zkutil::KeeperMultiException(code, ops, responses).getPathForFirstFailedOp() << std::endl;
}
catch (...)
{

8
dbms/src/Common/ZooKeeper/tests/zkutil_test_async.cpp
View File

@ -23,12 +23,12 @@ try
{
while (true)
{
std::vector<zkutil::ZooKeeper::TryGetFuture> futures;
std::vector<std::future<zkutil::GetResponse>> futures;
for (auto & node : nodes)
futures.push_back(zookeeper.asyncTryGet("/tmp/" + node));
futures.push_back(zookeeper.asyncGet("/tmp/" + node));
for (auto & future : futures)
std::cerr << (future.get().value.empty() ? ',' : '.');
std::cerr << (future.get().data.empty() ? ',' : '.');
}
});
}
@ -41,5 +41,5 @@ try
catch (const Poco::Exception & e)
{
std::cout << e.message() << std::endl;
throw;
return 1;
}

76
dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
View File

@ -1,41 +1,55 @@
#include <Common/ZooKeeper/ZooKeeper.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/typeid_cast.h>
#include <iostream>
#include <unistd.h>
#include <port/unistd.h>
using namespace zkutil;
int main()
int main(int argc, char ** argv)
try
{
try
if (argc < 2)
{
ZooKeeper zk("mtfilter01t:2181,metrika-test:2181,mtweb01t:2181", "", 5000);
Strings children;
std::cout << "create path" << std::endl;
zk.create("/test", "old", zkutil::CreateMode::Persistent);
zkutil::Stat stat;
zkutil::EventPtr watch = std::make_shared<Poco::Event>();
std::cout << "get path" << std::endl;
zk.get("/test", &stat, watch);
std::cout << "set path" << std::endl;
zk.set("/test", "new");
watch->wait();
std::cout << "watch happened" << std::endl;
std::cout << "remove path" << std::endl;
zk.remove("/test");
Ops ops;
ops.emplace_back(std::make_unique<Op::Create>("/test", "multi1", zk.getDefaultACL(), CreateMode::Persistent));
ops.emplace_back(std::make_unique<Op::SetData>("/test", "multi2", -1));
ops.emplace_back(std::make_unique<Op::Remove>("/test", -1));
std::cout << "multi" << std::endl;
OpResultsPtr res = zk.multi(ops);
std::cout << "path created: " << dynamic_cast<Op::Create &>(*ops[0]).getPathCreated() << std::endl;
}
catch (KeeperException & e)
{
std::cerr << "KeeperException " << e.what() << " " << e.message() << std::endl;
std::cerr << "Usage: ./zkutil_test_commands host:port,host:port...\n";
return 1;
}
ZooKeeper zk(argv[1], "", 5000);
std::cout << "create path" << std::endl;
zk.create("/test", "old", zkutil::CreateMode::Persistent);
zkutil::Stat stat;
zkutil::EventPtr watch = std::make_shared<Poco::Event>();
std::cout << "get path" << std::endl;
zk.get("/test", &stat, watch);
std::cout << "set path" << std::endl;
zk.set("/test", "new");
watch->wait();
std::cout << "watch happened" << std::endl;
std::cout << "remove path" << std::endl;
std::cout << "list path" << std::endl;
Strings children = zk.getChildren("/");
for (const auto & name : children)
std::cerr << "\t" << name << "\n";
zk.remove("/test");
zkutil::Requests ops;
ops.emplace_back(zkutil::makeCreateRequest("/test", "multi1", CreateMode::Persistent));
ops.emplace_back(zkutil::makeSetRequest("/test", "multi2", -1));
ops.emplace_back(zkutil::makeRemoveRequest("/test", -1));
std::cout << "multi" << std::endl;
zkutil::Responses res = zk.multi(ops);
std::cout << "path created: " << typeid_cast<const CreateResponse &>(*res[0]).path_created << std::endl;
return 0;
}
catch (KeeperException & e)
{
std::cerr << "KeeperException " << e.what() << " " << e.message() << std::endl;
return 1;
}

199
dbms/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp Normal file
View File

@ -0,0 +1,199 @@
#include <Poco/ConsoleChannel.h>
#include <Poco/Logger.h>
#include <Poco/Event.h>
#include <Common/ZooKeeper/ZooKeeperImpl.h>
#include <Common/typeid_cast.h>
#include <iostream>
#include <boost/algorithm/string.hpp>
using namespace ZooKeeperImpl;
int main(int argc, char ** argv)
try
{
if (argc < 2)
{
std::cerr << "Usage: ./zkutil_test_commands_new_lib host:port,host:port...\n";
return 1;
}
Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
Poco::Logger::root().setChannel(channel);
Poco::Logger::root().setLevel("trace");
std::string addresses_arg = argv[1];
std::vector<std::string> addresses_strings;
boost::split(addresses_strings, addresses_arg, boost::is_any_of(","));
ZooKeeper::Addresses addresses;
addresses.reserve(addresses_strings.size());
for (const auto & address_string : addresses_strings)
addresses.emplace_back(address_string);
ZooKeeper zk(addresses, {}, {}, {}, {5, 0}, {0, 50000}, {0, 50000});
Poco::Event event(true);
std::cout << "create\n";
zk.create("/test", "old", false, false, {},
[&](const ZooKeeper::CreateResponse & response)
{
if (response.error)
std::cerr << "Error (create) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Created path: " << response.path_created << '\n';
//event.set();
});
//event.wait();
std::cout << "get\n";
zk.get("/test",
[&](const ZooKeeper::GetResponse & response)
{
if (response.error)
std::cerr << "Error (get) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Value: " << response.data << '\n';
//event.set();
},
[](const ZooKeeper::WatchResponse & response)
{
if (response.error)
std::cerr << "Watch (get) on /test, Error " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Watch (get) on /test, path: " << response.path << ", type: " << response.type << '\n';
});
//event.wait();
std::cout << "set\n";
zk.set("/test", "new", -1,
[&](const ZooKeeper::SetResponse & response)
{
if (response.error)
std::cerr << "Error (set) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Set\n";
//event.set();
});
//event.wait();
std::cout << "list\n";
zk.list("/",
[&](const ZooKeeper::ListResponse & response)
{
if (response.error)
std::cerr << "Error (list) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
{
std::cerr << "Children:\n";
for (const auto & name : response.names)
std::cerr << name << "\n";
}
//event.set();
},
[](const ZooKeeper::WatchResponse & response)
{
if (response.error)
std::cerr << "Watch (list) on /, Error " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Watch (list) on /, path: " << response.path << ", type: " << response.type << '\n';
});
//event.wait();
std::cout << "exists\n";
zk.exists("/test",
[&](const ZooKeeper::ExistsResponse & response)
{
if (response.error)
std::cerr << "Error (exists) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Exists\n";
//event.set();
},
[](const ZooKeeper::WatchResponse & response)
{
if (response.error)
std::cerr << "Watch (exists) on /test, Error " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Watch (exists) on /test, path: " << response.path << ", type: " << response.type << '\n';
});
//event.wait();
std::cout << "remove\n";
zk.remove("/test", -1, [&](const ZooKeeper::RemoveResponse & response)
{
if (response.error)
std::cerr << "Error (remove) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
std::cerr << "Removed\n";
//event.set();
});
//event.wait();
std::cout << "multi\n";
ZooKeeper::Requests ops;
{
ZooKeeper::CreateRequest create_request;
create_request.path = "/test";
create_request.data = "multi1";
ops.emplace_back(std::make_shared<ZooKeeper::CreateRequest>(std::move(create_request)));
}
{
ZooKeeper::SetRequest set_request;
set_request.path = "/test";
set_request.data = "multi2";
ops.emplace_back(std::make_shared<ZooKeeper::SetRequest>(std::move(set_request)));
}
{
ZooKeeper::RemoveRequest remove_request;
remove_request.path = "/test";
ops.emplace_back(std::make_shared<ZooKeeper::RemoveRequest>(std::move(remove_request)));
}
zk.multi(ops, [&](const ZooKeeper::MultiResponse & response)
{
if (response.error)
std::cerr << "Error (multi) " << response.error << ": " << ZooKeeper::errorMessage(response.error) << '\n';
else
{
for (const auto & elem : response.responses)
if (elem->error)
std::cerr << "Error (elem) " << elem->error << ": " << ZooKeeper::errorMessage(elem->error) << '\n';
std::cerr << "Created path: " << typeid_cast<const ZooKeeper::CreateResponse &>(*response.responses[0]).path_created << '\n';
}
event.set();
});
event.wait();
return 0;
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(__PRETTY_FUNCTION__) << '\n';
return 1;
}

26
dbms/src/Common/ZooKeeper/tests/zookeeper_impl.cpp Normal file
View File

@ -0,0 +1,26 @@
#include <Common/ZooKeeper/ZooKeeperImpl.h>
#include <iostream>
int main()
try
{
ZooKeeperImpl::ZooKeeper zookeeper({Poco::Net::SocketAddress{"localhost:2181"}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000});
zookeeper.create("/test", "hello", false, false, {}, [](const ZooKeeperImpl::ZooKeeper::CreateResponse & response)
{
if (response.error)
std::cerr << "Error " << response.error << ": " << ZooKeeperImpl::ZooKeeper::errorMessage(response.error) << "\n";
else
std::cerr << "Path created: " << response.path_created << "\n";
});
sleep(100);
return 0;
}
catch (...)
{
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
return 1;
}

1
dbms/src/Common/config.h.in
View File

@ -9,6 +9,7 @@
#cmakedefine01 USE_RDKAFKA
#cmakedefine01 USE_CAPNP
#cmakedefine01 USE_EMBEDDED_COMPILER
#cmakedefine01 Poco_SQLODBC_FOUND
#cmakedefine01 Poco_DataODBC_FOUND
#cmakedefine01 Poco_MongoDB_FOUND
#cmakedefine01 Poco_NetSSL_FOUND

2
dbms/src/Common/config_build.cpp.in
View File

@ -6,6 +6,7 @@ const char * auto_config_build[]
{
"VERSION_FULL", "@VERSION_FULL@",
"VERSION_DESCRIBE", "@VERSION_DESCRIBE@",
"VERSION_GITHASH", "@VERSION_GITHASH@",
"BUILD_DATE", "@BUILD_DATE@",
"BUILD_TYPE", "@CMAKE_BUILD_TYPE@",
"SYSTEM", "@CMAKE_SYSTEM@",
@ -34,6 +35,7 @@ const char * auto_config_build[]
"USE_VECTORCLASS", "@USE_VECTORCLASS@",
"USE_RDKAFKA", "@USE_RDKAFKA@",
"USE_CAPNP", "@USE_CAPNP@",
"USE_Poco_SQLODBC", "@Poco_SQLODBC_FOUND@",
"USE_Poco_DataODBC", "@Poco_DataODBC_FOUND@",
"USE_Poco_MongoDB", "@Poco_MongoDB_FOUND@",
"USE_Poco_NetSSL", "@Poco_NetSSL_FOUND@",

1
dbms/src/Common/config_version.h.in
View File

@ -16,3 +16,4 @@
#cmakedefine VERSION_STRING "@VERSION_STRING@"
#cmakedefine VERSION_FULL "@VERSION_FULL@"
#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
#cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"

5
dbms/src/Common/localBackup.cpp
View File

@ -1,13 +1,10 @@
#include <sys/stat.h>
#include <unistd.h>
#include <string>
#include <iostream>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
#include <Common/Exception.h>
#include <port/unistd.h>
namespace DB

4
dbms/src/Common/randomSeed.cpp
View File

@ -1,12 +1,10 @@
#include <time.h>
#include <unistd.h>
#include <port/unistd.h>
#include <sys/types.h>
#include <Common/Exception.h>
#include <Common/randomSeed.h>
#include <Common/SipHash.h>
#include <Core/Types.h>
#ifdef __APPLE__
#include <common/apple_rt.h>
#endif

4
dbms/src/Common/tests/compact_array.cpp
View File

@ -7,15 +7,13 @@
#include <Common/CompactArray.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h>
#include <boost/filesystem.hpp>
#include <string>
#include <iostream>
#include <fstream>
#include <stdexcept>
#include <unistd.h>
#include <cstdlib>
#include <port/unistd.h>
namespace fs = boost::filesystem;

2
dbms/src/Common/typeid_cast.h
View File

@ -6,7 +6,7 @@
#include <string>
#include <Common/Exception.h>
#include <Common/demangle.h>
#include <common/demangle.h>
namespace DB

2
dbms/src/Core/Block.cpp
View File

@ -312,7 +312,7 @@ MutableColumns Block::mutateColumns() const
size_t num_columns = data.size();
MutableColumns columns(num_columns);
for (size_t i = 0; i < num_columns; ++i)
columns[i] = data[i].column ? data[i].column->mutate() : data[i].type->createColumn();
columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn();
return columns;
}

2
dbms/src/Core/Block.h
View File

@ -109,7 +109,7 @@ public:
/** Get empty columns with the same types as in block. */
MutableColumns cloneEmptyColumns() const;
/** Get columns from block for mutation. */
/** Get columns from block for mutation. Columns in block will be nullptr. */
MutableColumns mutateColumns() const;
/** Replace columns in a block */

15
dbms/src/Core/Defines.h
View File

@ -60,6 +60,7 @@
#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
#define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
#define DBMS_TCP_PROTOCOL_VERSION 54226
@ -74,8 +75,18 @@
#define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
#define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
#define ALWAYS_INLINE __attribute__((__always_inline__))
#define NO_INLINE __attribute__((__noinline__))
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
#if defined(_MSC_VER)
#define ALWAYS_INLINE __forceinline
#define NO_INLINE static __declspec(noinline)
#define MAY_ALIAS
#else
#define ALWAYS_INLINE __attribute__((__always_inline__))
#define NO_INLINE __attribute__((__noinline__))
#define MAY_ALIAS __attribute__((__may_alias__))
#endif
#define PLATFORM_NOT_SUPPORTED "The only supported platforms are x86_64 and AArch64 (work in progress)"

12
dbms/src/Core/Field.h
View File

@ -198,14 +198,14 @@ public:
template <typename T> T & get()
{
using TWithoutRef = std::remove_reference_t<T>;
TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<TWithoutRef*>(&storage);
TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<TWithoutRef*>(&storage);
return *ptr;
};
template <typename T> const T & get() const
{
using TWithoutRef = std::remove_reference_t<T>;
const TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<const TWithoutRef*>(&storage);
const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<const TWithoutRef*>(&storage);
return *ptr;
};
@ -340,7 +340,7 @@ private:
void createConcrete(T && x)
{
using JustT = std::decay_t<T>;
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
new (ptr) JustT(std::forward<T>(x));
which = TypeToEnum<JustT>::value;
}
@ -350,7 +350,7 @@ private:
void assignConcrete(T && x)
{
using JustT = std::decay_t<T>;
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
*ptr = std::forward<T>(x);
}
@ -398,7 +398,7 @@ private:
void create(const char * data, size_t size)
{
String * __attribute__((__may_alias__)) ptr = reinterpret_cast<String*>(&storage);
String * MAY_ALIAS ptr = reinterpret_cast<String*>(&storage);
new (ptr) String(data, size);
which = Types::String;
}
@ -434,7 +434,7 @@ private:
template <typename T>
void destroy()
{
T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(&storage);
T * MAY_ALIAS ptr = reinterpret_cast<T*>(&storage);
ptr->~T();
}
};

2
dbms/src/Core/Protocol.h
View File

@ -116,7 +116,7 @@ namespace Protocol
};
/// Whether the ssl must be used.
enum class Encryption
enum class Secure
{
Disable = 0,
Enable = 1,

44
dbms/src/Core/iostream_debug_helpers.cpp
View File

@ -5,6 +5,7 @@
#include <Core/ColumnWithTypeAndName.h>
#include <Core/Field.h>
#include <Core/NamesAndTypes.h>
#include <Common/FieldVisitors.h>
#include <Common/COWPtr.h>
#include <DataStreams/IBlockInputStream.h>
#include <DataTypes/IDataType.h>
@ -13,19 +14,19 @@
#include <Interpreters/ExpressionAnalyzer.h>
#include <Parsers/IAST.h>
namespace DB
{
std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what)
{
stream << "IBlockInputStream(name = " << what.getName() << ")";
//what.dumpTree(stream); // todo: set const
return stream;
}
std::ostream & operator<<(std::ostream & stream, const Field & what)
{
stream << "Field(type = " << what.getTypeName() << ")";
stream << applyVisitor(FieldVisitorDump(), what);
return stream;
}
@ -46,7 +47,6 @@ std::ostream & operator<<(std::ostream & stream, const IStorage & what)
stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
<< what.getColumns().getAllPhysical().toString()
<< "}";
// isRemote supportsSampling supportsFinal supportsPrewhere
return stream;
}
@ -71,27 +71,10 @@ std::ostream & operator<<(std::ostream & stream, const Block & what)
return stream;
}
template <typename T>
std::ostream & printCOWPtr(std::ostream & stream, const typename COWPtr<T>::Ptr & what)
{
stream << "COWPtr::Ptr(" << what.get();
if (what)
stream << ", use_count = " << what->use_count();
stream << ") {";
if (what)
stream << *what;
else
stream << "nullptr";
stream << "}";
return stream;
}
std::ostream & operator<<(std::ostream & stream, const ColumnWithTypeAndName & what)
{
stream << "ColumnWithTypeAndName(name = " << what.name << ", type = " << what.type << ", column = ";
return printCOWPtr<IColumn>(stream, what.column) << ")";
return dumpValue(stream, what.column) << ")";
}
std::ostream & operator<<(std::ostream & stream, const IColumn & what)
@ -112,15 +95,6 @@ std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what
return stream;
}
std::ostream & operator<<(std::ostream & stream, const SubqueryForSet & what)
{
stream << "SubqueryForSet(source = " << what.source
// TODO: << ", set = " << what.set << ", join = " << what.join
<< ", table = " << what.table
<< ")";
return stream;
}
std::ostream & operator<<(std::ostream & stream, const IAST & what)
{
stream << "IAST{";
@ -129,14 +103,4 @@ std::ostream & operator<<(std::ostream & stream, const IAST & what)
return stream;
}
std::ostream & operator<<(std::ostream & stream, const ExpressionAnalyzer & what)
{
stream << "ExpressionAnalyzer{"
<< "hasAggregation=" << what.hasAggregation()
<< ", SubqueriesForSet=" << what.getSubqueriesForSets()
<< ", ExternalTables=" << what.getExternalTables()
<< "}";
return stream;
}
}

17
dbms/src/Core/iostream_debug_helpers.h
View File

@ -2,7 +2,7 @@
#include <iostream>
#include <Client/Connection.h>
#include <Common/PODArray.h>
namespace DB
{
@ -37,26 +37,11 @@ std::ostream & operator<<(std::ostream & stream, const ColumnWithTypeAndName & w
class IColumn;
std::ostream & operator<<(std::ostream & stream, const IColumn & what);
struct SubqueryForSet;
std::ostream & operator<<(std::ostream & stream, const SubqueryForSet & what);
class IAST;
std::ostream & operator<<(std::ostream & stream, const IAST & what);
class ExpressionAnalyzer;
std::ostream & operator<<(std::ostream & stream, const ExpressionAnalyzer & what);
std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what);
template <typename T, size_t INITIAL_SIZE, typename TAllocator, size_t pad_right_>
std::ostream & operator<<(std::ostream & stream, const PODArray<T, INITIAL_SIZE, TAllocator, pad_right_> & what)
{
stream << "PODArray(size = " << what.size() << ", capacity = " << what.capacity() << ")";
dumpContainer(stream, what);
return stream;
};
}
/// some operator<< should be declared before operator<<(... std::shared_ptr<>)

71
dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp
View File

@ -12,18 +12,46 @@ namespace ErrorCodes
}
AggregatingSortedBlockInputStream::AggregatingSortedBlockInputStream(
const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_)
{
/// Fill in the column numbers that need to be aggregated.
for (size_t i = 0; i < num_columns; ++i)
{
ColumnWithTypeAndName & column = header.safeGetByPosition(i);
/// We leave only states of aggregate functions.
if (!startsWith(column.type->getName(), "AggregateFunction"))
{
column_numbers_not_to_aggregate.push_back(i);
continue;
}
/// Included into PK?
SortDescription::const_iterator it = description.begin();
for (; it != description.end(); ++it)
if (it->column_name == column.name || (it->column_name.empty() && it->column_number == i))
break;
if (it != description.end())
{
column_numbers_not_to_aggregate.push_back(i);
continue;
}
column_numbers_to_aggregate.push_back(i);
}
}
Block AggregatingSortedBlockInputStream::readImpl()
{
if (finished)
return Block();
if (children.size() == 1)
return children[0]->read();
Block header;
MutableColumns merged_columns;
init(header, merged_columns);
init(merged_columns);
if (has_collation)
throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR);
@ -31,37 +59,6 @@ Block AggregatingSortedBlockInputStream::readImpl()
if (merged_columns.empty())
return Block();
/// Additional initialization.
if (next_key.empty())
{
/// Fill in the column numbers that need to be aggregated.
for (size_t i = 0; i < num_columns; ++i)
{
ColumnWithTypeAndName & column = header.safeGetByPosition(i);
/// We leave only states of aggregate functions.
if (!startsWith(column.type->getName(), "AggregateFunction"))
{
column_numbers_not_to_aggregate.push_back(i);
continue;
}
/// Included into PK?
SortDescription::const_iterator it = description.begin();
for (; it != description.end(); ++it)
if (it->column_name == column.name || (it->column_name.empty() && it->column_number == i))
break;
if (it != description.end())
{
column_numbers_not_to_aggregate.push_back(i);
continue;
}
column_numbers_to_aggregate.push_back(i);
}
}
columns_to_aggregate.resize(column_numbers_to_aggregate.size());
for (size_t i = 0, size = columns_to_aggregate.size(); i < size; ++i)
columns_to_aggregate[i] = typeid_cast<ColumnAggregateFunction *>(merged_columns[column_numbers_to_aggregate[i]].get());

6
dbms/src/DataStreams/AggregatingSortedBlockInputStream.h
View File

@ -21,10 +21,8 @@ namespace DB
class AggregatingSortedBlockInputStream : public MergingSortedBlockInputStream
{
public:
AggregatingSortedBlockInputStream(BlockInputStreams inputs_, const SortDescription & description_, size_t max_block_size_)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_)
{
}
AggregatingSortedBlockInputStream(
const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_);
String getName() const override { return "AggregatingSorted"; }

12
dbms/src/DataStreams/CollapsingSortedBlockInputStream.cpp
View File

@ -108,13 +108,8 @@ Block CollapsingSortedBlockInputStream::readImpl()
if (finished)
return {};
if (children.size() == 1)
return children[0]->read();
Block header;
MutableColumns merged_columns;
init(header, merged_columns);
init(merged_columns);
if (has_collation)
throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR);
@ -122,11 +117,6 @@ Block CollapsingSortedBlockInputStream::readImpl()
if (merged_columns.empty())
return {};
/// Additional initialization.
if (first_negative.empty())
sign_column_number = header.getPositionByName(sign_column);
merge(merged_columns, queue);
return header.cloneWithColumns(std::move(merged_columns));
}

7
dbms/src/DataStreams/CollapsingSortedBlockInputStream.h
View File

@ -25,10 +25,10 @@ class CollapsingSortedBlockInputStream : public MergingSortedBlockInputStream
public:
CollapsingSortedBlockInputStream(
BlockInputStreams inputs_, const SortDescription & description_,
const String & sign_column_, size_t max_block_size_, WriteBuffer * out_row_sources_buf_ = nullptr)
const String & sign_column, size_t max_block_size_, WriteBuffer * out_row_sources_buf_ = nullptr)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_, 0, out_row_sources_buf_)
, sign_column(sign_column_)
{
sign_column_number = header.getPositionByName(sign_column);
}
String getName() const override { return "CollapsingSorted"; }
@ -38,8 +38,7 @@ protected:
Block readImpl() override;
private:
String sign_column;
size_t sign_column_number = 0;
size_t sign_column_number;
Logger * log = &Logger::get("CollapsingSortedBlockInputStream");

2
dbms/src/DataStreams/ColumnGathererStream.cpp
View File

@ -76,7 +76,7 @@ Block ColumnGathererStream::readImpl()
return Block();
output_block = Block{column.cloneEmpty()};
MutableColumnPtr output_column = output_block.getByPosition(0).column->mutate();
MutableColumnPtr output_column = output_block.getByPosition(0).column->assumeMutable();
output_column->gather(*this);
if (!output_column->empty())
output_block.getByPosition(0).column = std::move(output_column);

50
dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp
View File

@ -12,6 +12,31 @@ namespace ErrorCodes
}
GraphiteRollupSortedBlockInputStream::GraphiteRollupSortedBlockInputStream(
const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_,
const Graphite::Params & params, time_t time_of_merge)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_),
params(params), time_of_merge(time_of_merge)
{
size_t max_size_of_aggregate_state = 0;
for (const auto & pattern : params.patterns)
if (pattern.function->sizeOfData() > max_size_of_aggregate_state)
max_size_of_aggregate_state = pattern.function->sizeOfData();
place_for_aggregate_state.resize(max_size_of_aggregate_state);
/// Memoize column numbers in block.
path_column_num = header.getPositionByName(params.path_column_name);
time_column_num = header.getPositionByName(params.time_column_name);
value_column_num = header.getPositionByName(params.value_column_name);
version_column_num = header.getPositionByName(params.version_column_name);
for (size_t i = 0; i < num_columns; ++i)
if (i != time_column_num && i != value_column_num && i != version_column_num)
unmodified_column_numbers.push_back(i);
}
const Graphite::Pattern * GraphiteRollupSortedBlockInputStream::selectPatternForPath(StringRef path) const
{
for (const auto & pattern : params.patterns)
@ -68,10 +93,8 @@ Block GraphiteRollupSortedBlockInputStream::readImpl()
if (finished)
return Block();
Block header;
MutableColumns merged_columns;
init(header, merged_columns);
init(merged_columns);
if (has_collation)
throw Exception("Logical error: " + getName() + " does not support collations", ErrorCodes::LOGICAL_ERROR);
@ -79,27 +102,6 @@ Block GraphiteRollupSortedBlockInputStream::readImpl()
if (merged_columns.empty())
return Block();
/// Additional initialization.
if (is_first)
{
size_t max_size_of_aggregate_state = 0;
for (const auto & pattern : params.patterns)
if (pattern.function->sizeOfData() > max_size_of_aggregate_state)
max_size_of_aggregate_state = pattern.function->sizeOfData();
place_for_aggregate_state.resize(max_size_of_aggregate_state);
/// Memoize column numbers in block.
path_column_num = header.getPositionByName(params.path_column_name);
time_column_num = header.getPositionByName(params.time_column_name);
value_column_num = header.getPositionByName(params.value_column_name);
version_column_num = header.getPositionByName(params.version_column_name);
for (size_t i = 0; i < num_columns; ++i)
if (i != time_column_num && i != value_column_num && i != version_column_num)
unmodified_column_numbers.push_back(i);
}
merge(merged_columns, queue);
return header.cloneWithColumns(std::move(merged_columns));
}

8
dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h
View File

@ -126,12 +126,8 @@ class GraphiteRollupSortedBlockInputStream : public MergingSortedBlockInputStrea
{
public:
GraphiteRollupSortedBlockInputStream(
BlockInputStreams inputs_, const SortDescription & description_, size_t max_block_size_,
const Graphite::Params & params, time_t time_of_merge)
: MergingSortedBlockInputStream(inputs_, description_, max_block_size_),
params(params), time_of_merge(time_of_merge)
{
}
const BlockInputStreams & inputs_, const SortDescription & description_, size_t max_block_size_,
const Graphite::Params & params, time_t time_of_merge);
String getName() const override { return "GraphiteRollupSorted"; }

Some files were not shown because too many files have changed in this diff Show More