mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-11 17:02:25 +00:00
Merge branch 'master' into data-type-with-dictionary
This commit is contained in:
Nikolai Kochetov
commit
c6a3fec726
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -1,6 +1,3 @@
|
||||
[submodule "contrib/zookeeper"]
|
||||
path = contrib/zookeeper
|
||||
url = https://github.com/ClickHouse-Extras/zookeeper.git
|
||||
[submodule "contrib/poco"]
|
||||
path = contrib/poco
|
||||
url = https://github.com/ClickHouse-Extras/poco
|
||||
|
||||
@ -13,7 +13,7 @@ matrix:
|
||||
# apt:
|
||||
# sources:
|
||||
# - ubuntu-toolchain-r-test
|
||||
# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
|
||||
# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ]
|
||||
#
|
||||
# env:
|
||||
# - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7"
|
||||
@ -36,7 +36,7 @@ matrix:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
- llvm-toolchain-trusty-5.0
|
||||
packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
|
||||
packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
|
||||
|
||||
env:
|
||||
- MATRIX_EVAL="export CC=clang-5.0 && export CXX=clang++-5.0"
|
||||
|
||||
29
CHANGELOG.md
29
CHANGELOG.md
@ -1,4 +1,31 @@
|
||||
# ClickHouse 1.1.54356 Release Candidate, 2018-03-06
|
||||
# ClickHouse release 1.1.54370, 2018-03-16
|
||||
|
||||
## New features:
|
||||
|
||||
* Added the `system.macros` table and auto updating of macros when the config file is changed.
|
||||
* Added the `SYSTEM RELOAD CONFIG` query.
|
||||
* Added the `maxIntersections(left_col, right_col)` aggregate function, which returns the maximum number of simultaneously intersecting intervals `[left; right]`. The `maxIntersectionsPosition(left, right)` function returns the beginning of the "maximum" interval. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
|
||||
|
||||
## Improvements:
|
||||
|
||||
* When inserting data in a `Replicated` table, fewer requests are made to `ZooKeeper` (and most of the user-level errors have disappeared from the `ZooKeeper` log).
|
||||
* Added the ability to create aliases for sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
|
||||
|
||||
## Bug fixes:
|
||||
|
||||
* Fixed the `Illegal PREWHERE` error when reading from `Merge` tables over `Distributed` tables.
|
||||
* Added fixes that allow you to run `clickhouse-server` in IPv4-only Docker containers.
|
||||
* Fixed a race condition when reading from system `system.parts_columns` tables.
|
||||
* Removed double buffering during a synchronous insert to a `Distributed` table, which could have caused the connection to timeout.
|
||||
* Fixed a bug that caused excessively long waits for an unavailable replica before beginning a `SELECT` query.
|
||||
* Fixed incorrect dates in the `system.parts` table.
|
||||
* Fixed a bug that made it impossible to insert data in a `Replicated` table if `chroot` was non-empty in the configuration of the `ZooKeeper` cluster.
|
||||
* Fixed the vertical merging algorithm for an empty `ORDER BY` table.
|
||||
* Restored the ability to use dictionaries in queries to remote tables, even if these dictionaries are not present on the requestor server. This functionality was lost in release 1.1.54362.
|
||||
* Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side argument of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358.
|
||||
* Removed extraneous error-level logging of `Not found column ... in block`.
|
||||
|
||||
# ClickHouse release 1.1.54356, 2018-03-06
|
||||
|
||||
## New features:
|
||||
|
||||
|
||||
@ -1,4 +1,32 @@
|
||||
# ClickHouse 1.1.54362 Release Candidate, 2018-03-11
|
||||
# ClickHouse release 1.1.54370, 2018-03-16
|
||||
|
||||
## Новые возможности:
|
||||
|
||||
* Добавлена системная таблица `system.macros` и автоматическое обновление макросов при изменении конфигурационного файла.
|
||||
* Добавлен запрос `SYSTEM RELOAD CONFIG`.
|
||||
* Добавлена агрегатная функция `maxIntersections(left_col, right_col)`, возвращающая максимальное количество одновременно пересекающихся интервалов `[left; right]`. Функция `maxIntersectionsPosition(left, right)` возвращает начало такого "максимального" интервала. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
|
||||
|
||||
## Улучшения:
|
||||
|
||||
* При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок).
|
||||
* Добавлена возможность создавать алиасы для множеств. Пример: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
|
||||
|
||||
## Исправление ошибок:
|
||||
|
||||
* Исправлена ошибка `Illegal PREWHERE` при чтении из Merge-таблицы над `Distributed`-таблицами.
|
||||
* Добавлены исправления, позволяющие запускать clickhouse-server в IPv4-only docker-контейнерах.
|
||||
* Исправлен race condition при чтении из системной таблицы `system.parts_columns`
|
||||
* Убрана двойная буферизация при синхронной вставке в `Distributed`-таблицу, которая могла приводить к timeout-ам соединений.
|
||||
* Исправлена ошибка, вызывающая чрезмерно долгое ожидание недоступной реплики перед началом выполнения `SELECT`.
|
||||
* Исправлено некорректное отображение дат в таблице `system.parts`.
|
||||
* Исправлена ошибка, приводящая к невозможности вставить данные в `Replicated`-таблицу, если в конфигурации кластера `ZooKeeper` задан непустой `chroot`.
|
||||
* Исправлен алгоритм вертикального мержа при пустом ключе `ORDER BY` таблицы.
|
||||
* Возвращена возможность использовать словари в запросах к удаленным таблицам, даже если этих словарей нет на сервере-инициаторе. Данная функциональность была потеряна в версии 1.1.54362.
|
||||
* Восстановлено поведение, при котором в запросах типа `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` в правой части `IN` должна использоваться удаленная таблица `default.table`, а не локальная. данное поведение было нарушено в версии 1.1.54358.
|
||||
* Устранено ненужное Error-level логирование `Not found column ... in block`.
|
||||
|
||||
|
||||
# Релиз ClickHouse 1.1.54362, 2018-03-11
|
||||
|
||||
## Новые возможности:
|
||||
|
||||
|
||||
@ -252,13 +252,13 @@ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake)
|
||||
else ()
|
||||
include (cmake/find_odbc.cmake)
|
||||
endif ()
|
||||
message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}")
|
||||
include (cmake/find_poco.cmake)
|
||||
include (cmake/find_lz4.cmake)
|
||||
include (cmake/find_sparsehash.cmake)
|
||||
include (cmake/find_rt.cmake)
|
||||
include (cmake/find_execinfo.cmake)
|
||||
include (cmake/find_readline_edit.cmake)
|
||||
include (cmake/find_zookeeper.cmake)
|
||||
include (cmake/find_re2.cmake)
|
||||
include (cmake/find_rdkafka.cmake)
|
||||
include (cmake/find_capnp.cmake)
|
||||
|
||||
@ -1,32 +1,34 @@
|
||||
# Contributing to ClickHouse
|
||||
|
||||
## Technical info
|
||||
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
|
||||
https://clickhouse.yandex/docs/en/development/index.html
|
||||
|
||||
## Legal info
|
||||
|
||||
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
|
||||
1) https://yandex.ru/legal/cla/?lang=en (in English) and
|
||||
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
|
||||
|
||||
By adopting the CLA, you state the following:
|
||||
|
||||
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
|
||||
* You has read the terms and conditions of the CLA and agree with them in full,
|
||||
* You are legally able to provide and license your contributions as stated,
|
||||
* We may use your contributions for our open source projects and for any other our project too,
|
||||
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
|
||||
|
||||
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
|
||||
|
||||
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
|
||||
|
||||
```
|
||||
I hereby agree to the terms of the CLA available at: [link].
|
||||
```
|
||||
|
||||
Replace the bracketed text as follows:
|
||||
* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).
|
||||
|
||||
It is enough to provide us such notification at once.
|
||||
# Contributing to ClickHouse
|
||||
|
||||
## Technical info
|
||||
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
|
||||
https://clickhouse.yandex/docs/en/development/index.html
|
||||
|
||||
## Legal info
|
||||
|
||||
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
|
||||
1) https://yandex.ru/legal/cla/?lang=en (in English) and
|
||||
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
|
||||
|
||||
By adopting the CLA, you state the following:
|
||||
|
||||
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
|
||||
* You has read the terms and conditions of the CLA and agree with them in full,
|
||||
* You are legally able to provide and license your contributions as stated,
|
||||
* We may use your contributions for our open source projects and for any other our project too,
|
||||
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
|
||||
|
||||
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
|
||||
|
||||
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
|
||||
|
||||
```
|
||||
I hereby agree to the terms of the CLA available at: [link].
|
||||
```
|
||||
|
||||
Replace the bracketed text as follows:
|
||||
* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian).
|
||||
|
||||
It is enough to provide us such notification at once.
|
||||
|
||||
If you don't agree with the CLA, you still can open a pull request to provide your contributions.
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
# ClickHouse
|
||||
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
|
||||
|
||||
[Read more...](https://clickhouse.yandex/)
|
||||
🎤🥂 **ClickHouse Meetup in [Sunnyvale](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/248898966/) & [San Francisco](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/249162518/), April 23-27** 🍰🔥🐻
|
||||
|
||||
Learn more about ClickHouse at [https://clickhouse.yandex/](https://clickhouse.yandex/)
|
||||
|
||||
[](https://travis-ci.org/yandex/ClickHouse)
|
||||
|
||||
@ -51,6 +51,7 @@
|
||||
|
||||
set(Poco_HINTS
|
||||
/usr/local
|
||||
/usr/local/include/Poco
|
||||
C:/AppliedInformatics
|
||||
${Poco_DIR}
|
||||
$ENV{Poco_DIR}
|
||||
@ -230,5 +231,3 @@ if(${Poco_OSP_FOUND})
|
||||
endif()
|
||||
|
||||
message(STATUS "Found Poco: ${Poco_LIBRARIES}")
|
||||
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt")
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
find_package (Poco COMPONENTS Net NetSSL XML Data Crypto DataODBC MongoDB)
|
||||
find_package (Poco COMPONENTS Net NetSSL XML SQL Data Crypto DataODBC MongoDB)
|
||||
endif ()
|
||||
|
||||
if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
|
||||
@ -24,6 +24,15 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
|
||||
set (ENABLE_DATA_SQLITE 0 CACHE BOOL "")
|
||||
set (ENABLE_DATA_MYSQL 0 CACHE BOOL "")
|
||||
set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "")
|
||||
# new after 2.0.0:
|
||||
set (POCO_ENABLE_ZIP 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_PAGECOMPILER 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_PAGECOMPILER_FILE2PAGE 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_REDIS 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_SQL_SQLITE 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_SQL_MYSQL 0 CACHE BOOL "")
|
||||
set (POCO_ENABLE_SQL_POSTGRESQL 0 CACHE BOOL "")
|
||||
|
||||
set (POCO_UNBUNDLED 1 CACHE BOOL "")
|
||||
set (POCO_UNBUNDLED_PCRE 0 CACHE BOOL "")
|
||||
set (POCO_UNBUNDLED_EXPAT 0 CACHE BOOL "")
|
||||
@ -43,13 +52,39 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
|
||||
set (Poco_MongoDB_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/")
|
||||
endif ()
|
||||
|
||||
if (ODBC_FOUND)
|
||||
set (Poco_DataODBC_FOUND 1)
|
||||
set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
|
||||
set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
|
||||
|
||||
if (EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/")
|
||||
set (Poco_SQL_FOUND 1)
|
||||
set (Poco_SQL_LIBRARY PocoSQL)
|
||||
set (Poco_SQL_INCLUDE_DIRS
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/include"
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include"
|
||||
)
|
||||
if (ODBC_FOUND)
|
||||
set (Poco_SQLODBC_FOUND 1)
|
||||
set (Poco_SQLODBC_INCLUDE_DIRS
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/"
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
|
||||
${ODBC_INCLUDE_DIRECTORIES}
|
||||
)
|
||||
set (Poco_SQLODBC_LIBRARY PocoSQLODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
|
||||
endif ()
|
||||
else ()
|
||||
set (Poco_Data_FOUND 1)
|
||||
set (Poco_Data_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include")
|
||||
set (Poco_Data_LIBRARY PocoData)
|
||||
if (ODBC_FOUND)
|
||||
set (Poco_DataODBC_FOUND 1)
|
||||
set (Poco_DataODBC_INCLUDE_DIRS
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
|
||||
${ODBC_INCLUDE_DIRECTORIES}
|
||||
)
|
||||
set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (OPENSSL_FOUND)
|
||||
# TODO! fix internal ssl
|
||||
if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
|
||||
set (Poco_NetSSL_FOUND 1)
|
||||
set (Poco_NetSSL_LIBRARY PocoNetSSL)
|
||||
set (Poco_Crypto_LIBRARY PocoCrypto)
|
||||
@ -65,7 +100,6 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
|
||||
set (Poco_Foundation_LIBRARY PocoFoundation)
|
||||
set (Poco_Util_LIBRARY PocoUtil)
|
||||
set (Poco_Net_LIBRARY PocoNet)
|
||||
set (Poco_Data_LIBRARY PocoData)
|
||||
set (Poco_XML_LIBRARY PocoXML)
|
||||
endif ()
|
||||
|
||||
@ -85,3 +119,5 @@ message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${
|
||||
# ClickHouse-Extras/clickhouse_warning
|
||||
# ClickHouse-Extras/clickhouse-purge-logs-on-no-space
|
||||
# ClickHouse-Extras/clickhouse_freebsd
|
||||
# ClickHouse-Extras/clikhouse_no_zlib
|
||||
# ClickHouse-Extras/clickhouse-fix-atomic
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
if (ARCH_ARM)
|
||||
# bundled have some asm broken for arm, use package libzookeeper-mt-dev
|
||||
set(USE_INTERNAL_ZOOKEEPER_LIBRARY 0 CACHE BOOL "")
|
||||
endif ()
|
||||
|
||||
option (USE_INTERNAL_ZOOKEEPER_LIBRARY "Set to FALSE to use system zookeeper library instead of bundled" ${NOT_UNBUNDLED})
|
||||
|
||||
if (USE_INTERNAL_ZOOKEEPER_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zookeeper/src/c/CMakeLists.txt")
|
||||
message (WARNING "submodule contrib/zookeeper is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
set (USE_INTERNAL_ZOOKEEPER_LIBRARY 0)
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
find_library (ZOOKEEPER_LIBRARY zookeeper_mt)
|
||||
find_path (ZOOKEEPER_INCLUDE_DIR NAMES zookeeper/zookeeper.h PATHS ${ZOOKEEPER_INCLUDE_PATHS})
|
||||
set(ZOOKEEPER_INCLUDE_DIR "${ZOOKEEPER_INCLUDE_DIR}/zookeeper")
|
||||
endif ()
|
||||
|
||||
if (ZOOKEEPER_LIBRARY AND ZOOKEEPER_INCLUDE_DIR)
|
||||
else ()
|
||||
set (USE_INTERNAL_ZOOKEEPER_LIBRARY 1)
|
||||
set(WANT_CPPUNIT 0 CACHE BOOL "")
|
||||
set (ZOOKEEPER_LIBRARY zookeeper)
|
||||
endif ()
|
||||
|
||||
message (STATUS "Using zookeeper: ${ZOOKEEPER_INCLUDE_DIR} : ${ZOOKEEPER_LIBRARY}")
|
||||
|
||||
|
||||
# how to make clickhouse branch of https://github.com/ClickHouse-Extras/zookeeper.git :
|
||||
# git remote add upstream https://github.com/apache/zookeeper.git
|
||||
# git checkhout upstream/master
|
||||
# git branch -D clickhouse
|
||||
# git checkout -b clickhouse
|
||||
# git merge clickhouse_misc
|
||||
# git merge clickhouse_706
|
||||
24
contrib/CMakeLists.txt
vendored
24
contrib/CMakeLists.txt
vendored
@ -25,10 +25,6 @@ if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
|
||||
add_subdirectory (double-conversion)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
add_subdirectory (zookeeper/src/c)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_CITYHASH_LIBRARY)
|
||||
add_subdirectory (libcityhash)
|
||||
endif ()
|
||||
@ -100,13 +96,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
|
||||
mark_as_advanced (ZLIB_INCLUDE_DIR)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
|
||||
set (WITH_BUNDLED_SSL 1)
|
||||
if (MAKE_STATIC_LIBRARIES)
|
||||
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
|
||||
set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
|
||||
else ()
|
||||
set (WITH_SSL 0 CACHE INTERNAL "")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
add_subdirectory (librdkafka)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
|
||||
@ -127,16 +127,18 @@ endif ()
|
||||
|
||||
|
||||
if (USE_INTERNAL_POCO_LIBRARY)
|
||||
set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
|
||||
set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
|
||||
set (_save ${ENABLE_TESTS})
|
||||
set (ENABLE_TESTS 0)
|
||||
set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
set (DISABLE_INTERNAL_OPENSSL 1)
|
||||
set (ENABLE_NETSSL 0) # TODO!
|
||||
set (ENABLE_CRYPTO 0) # TODO!
|
||||
set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
|
||||
set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
|
||||
set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
|
||||
endif ()
|
||||
if (MSVC)
|
||||
set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
|
||||
endif ()
|
||||
add_subdirectory (poco)
|
||||
unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)
|
||||
|
||||
2
contrib/boost
vendored
2
contrib/boost
vendored
@ -1 +1 @@
|
||||
Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e
|
||||
Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded
|
||||
@ -5,7 +5,11 @@
|
||||
/* #undef AC_APPLE_UNIVERSAL_BUILD */
|
||||
|
||||
/* Define to 1 if the compiler supports __builtin_expect. */
|
||||
#if _MSC_VER
|
||||
#define HAVE_BUILTIN_EXPECT 0
|
||||
#else
|
||||
#define HAVE_BUILTIN_EXPECT 1
|
||||
#endif
|
||||
|
||||
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||
#define HAVE_DLFCN_H 1
|
||||
|
||||
@ -2,4 +2,8 @@ add_library(farmhash
|
||||
farmhash.cc
|
||||
farmhash.h)
|
||||
|
||||
if (MSVC)
|
||||
target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1)
|
||||
endif ()
|
||||
|
||||
target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
@ -13,7 +13,11 @@
|
||||
/* #undef ENABLE_SIZED_DELETE */
|
||||
|
||||
/* Define to 1 if compiler supports __builtin_expect */
|
||||
#if _MSC_VER
|
||||
#define HAVE_BUILTIN_EXPECT 0
|
||||
#else
|
||||
#define HAVE_BUILTIN_EXPECT 1
|
||||
#endif
|
||||
|
||||
/* Define to 1 if compiler supports __builtin_stack_pointer */
|
||||
/* #undef HAVE_BUILTIN_STACK_POINTER */
|
||||
|
||||
2
contrib/poco
vendored
2
contrib/poco
vendored
@ -1 +1 @@
|
||||
Subproject commit 8238852d7ab2a4abdf87adff233b3b83686f4fe4
|
||||
Subproject commit 930a7ec1154f4f9711edfb4b4a39f9fff2a5bbb5
|
||||
1
contrib/zookeeper
vendored
1
contrib/zookeeper
vendored
@ -1 +0,0 @@
|
||||
Subproject commit 438afae5af36c5be9c82d074f43a9bb19e0797c0
|
||||
2
contrib/zstd
vendored
2
contrib/zstd
vendored
@ -1 +1 @@
|
||||
Subproject commit f4340f46b2387bc8de7d5320c0b83bb1499933ad
|
||||
Subproject commit 255597502c3a4ef150abc964e376d4202a8c2929
|
||||
@ -8,7 +8,7 @@
|
||||
# Требуется clang, желательно наиболее свежий (trunk).
|
||||
#
|
||||
# Используется при сборке пакетов.
|
||||
# Заголовочные файлы записываются в пакет clickhouse-server-base, в директорию /usr/share/clickhouse/headers.
|
||||
# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers.
|
||||
#
|
||||
# Если вы хотите установить их самостоятельно, без сборки пакета,
|
||||
# чтобы clickhouse-server видел их там, где ожидается, выполните:
|
||||
|
||||
@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io
|
||||
${LINK_LIBRARIES_ONLY_ON_X86_64}
|
||||
${LZ4_LIBRARY}
|
||||
${ZSTD_LIBRARY}
|
||||
${ZOOKEEPER_LIBRARY}
|
||||
${DOUBLE_CONVERSION_LIBRARIES}
|
||||
${Poco_Net_LIBRARY}
|
||||
${Poco_Data_LIBRARY}
|
||||
@ -146,8 +145,6 @@ target_link_libraries (dbms
|
||||
clickhouse_common_config
|
||||
clickhouse_common_io
|
||||
${MYSQLXX_LIBRARY}
|
||||
${FARMHASH_LIBRARIES}
|
||||
${METROHASH_LIBRARIES}
|
||||
${RE2_LIBRARY}
|
||||
${RE2_ST_LIBRARY}
|
||||
${OPENSSL_CRYPTO_LIBRARY}
|
||||
@ -158,19 +155,33 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
|
||||
target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_BOOST_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
endif ()
|
||||
|
||||
if (Poco_DataODBC_FOUND)
|
||||
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
|
||||
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES})
|
||||
if (Poco_SQLODBC_FOUND)
|
||||
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
|
||||
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
|
||||
if (NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIRS})
|
||||
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS} PUBLIC ${Poco_SQL_INCLUDE_DIRS})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
|
||||
target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
if (Poco_DataODBC_FOUND)
|
||||
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
|
||||
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
|
||||
if (NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
if (Poco_MongoDB_FOUND)
|
||||
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
|
||||
endif()
|
||||
@ -217,6 +228,7 @@ endif ()
|
||||
target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
|
||||
target_include_directories (clickhouse_common_io PUBLIC ${Poco_DataODBC_INCLUDE_DIRS})
|
||||
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
|
||||
|
||||
# also for copy_headers.sh:
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
set(VERSION_DESCRIBE v1.1.54362-testing)
|
||||
set(VERSION_REVISION 54362)
|
||||
set(VERSION_DESCRIBE v1.1.54378-testing)
|
||||
set(VERSION_REVISION 54378)
|
||||
set(VERSION_GITHASH 5b19d89133a5ff7c72e40cc8c0226cb00466ba10)
|
||||
# end of autochange
|
||||
|
||||
set (VERSION_MAJOR 1)
|
||||
|
||||
@ -70,7 +70,10 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
|
||||
return combinator->transformAggregateFunction(nested_function, argument_types, parameters);
|
||||
}
|
||||
|
||||
return getImpl(name, argument_types, parameters, recursion_level);
|
||||
auto res = getImpl(name, argument_types, parameters, recursion_level);
|
||||
if (!res)
|
||||
throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionMaxIntersections.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
AggregateFunctionPtr createAggregateFunctionMaxIntersections(
|
||||
AggregateFunctionIntersectionsKind kind,
|
||||
const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{
|
||||
assertBinary(name, argument_types);
|
||||
assertNoParameters(name, parameters);
|
||||
|
||||
AggregateFunctionPtr res(createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types));
|
||||
if (!res)
|
||||
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
|
||||
+ " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("maxIntersections", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{ return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Count, name, argument_types, parameters); });
|
||||
|
||||
factory.registerFunction("maxIntersectionsPosition", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{ return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Position, name, argument_types, parameters); });
|
||||
}
|
||||
|
||||
}
|
||||
170
dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
Normal file
170
dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
Normal file
@ -0,0 +1,170 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <Common/ArenaAllocator.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
|
||||
#define AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE 0xFFFFFF
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int TOO_LARGE_ARRAY_SIZE;
|
||||
}
|
||||
|
||||
|
||||
/** maxIntersections: returns maximum count of the intersected intervals defined by start_column and end_column values,
|
||||
* maxIntersectionsPosition: returns leftmost position of maximum intersection of intervals.
|
||||
*/
|
||||
|
||||
/// Similar to GroupArrayNumericData.
|
||||
template <typename T>
|
||||
struct MaxIntersectionsData
|
||||
{
|
||||
/// Left or right end of the interval and signed weight; with positive sign for begin of interval and negative sign for end of interval.
|
||||
using Value = std::pair<T, Int64>;
|
||||
|
||||
// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
|
||||
using Allocator = MixedArenaAllocator<4096>;
|
||||
using Array = PODArray<Value, 32, Allocator>;
|
||||
|
||||
Array value;
|
||||
};
|
||||
|
||||
enum class AggregateFunctionIntersectionsKind
|
||||
{
|
||||
Count,
|
||||
Position
|
||||
};
|
||||
|
||||
template <typename PointType>
|
||||
class AggregateFunctionIntersectionsMax final
|
||||
: public IAggregateFunctionDataHelper<MaxIntersectionsData<PointType>, AggregateFunctionIntersectionsMax<PointType>>
|
||||
{
|
||||
private:
|
||||
AggregateFunctionIntersectionsKind kind;
|
||||
|
||||
public:
|
||||
AggregateFunctionIntersectionsMax(AggregateFunctionIntersectionsKind kind_, const DataTypes & arguments)
|
||||
: kind(kind_)
|
||||
{
|
||||
if (!arguments[0]->isNumber())
|
||||
throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
if (!arguments[1]->isNumber())
|
||||
throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
if (!arguments[0]->equals(*arguments[1]))
|
||||
throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return kind == AggregateFunctionIntersectionsKind::Count
|
||||
? "maxIntersections"
|
||||
: "maxIntersectionsPosition";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
if (kind == AggregateFunctionIntersectionsKind::Count)
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
else
|
||||
return std::make_shared<DataTypeNumber<PointType>>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
|
||||
{
|
||||
PointType left = static_cast<const ColumnVector<PointType> &>(*columns[0]).getData()[row_num];
|
||||
PointType right = static_cast<const ColumnVector<PointType> &>(*columns[1]).getData()[row_num];
|
||||
|
||||
if (!isNaN(left))
|
||||
this->data(place).value.push_back(std::make_pair(left, Int64(1)), arena);
|
||||
|
||||
if (!isNaN(right))
|
||||
this->data(place).value.push_back(std::make_pair(right, Int64(-1)), arena);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
|
||||
{
|
||||
auto & cur_elems = this->data(place);
|
||||
auto & rhs_elems = this->data(rhs);
|
||||
|
||||
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
const auto & value = this->data(place).value;
|
||||
size_t size = value.size();
|
||||
writeVarUInt(size, buf);
|
||||
buf.write(reinterpret_cast<const char *>(&value[0]), size * sizeof(value[0]));
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
|
||||
{
|
||||
size_t size = 0;
|
||||
readVarUInt(size, buf);
|
||||
|
||||
if (unlikely(size > AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE))
|
||||
throw Exception("Too large array size", ErrorCodes::TOO_LARGE_ARRAY_SIZE);
|
||||
|
||||
auto & value = this->data(place).value;
|
||||
|
||||
value.resize(size, arena);
|
||||
buf.read(reinterpret_cast<char *>(&value[0]), size * sizeof(value[0]));
|
||||
}
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
Int64 current_intersections = 0;
|
||||
Int64 max_intersections = 0;
|
||||
PointType position_of_max_intersections = 0;
|
||||
|
||||
/// const_cast because we will sort the array
|
||||
auto & array = const_cast<typename MaxIntersectionsData<PointType>::Array &>(this->data(place).value);
|
||||
|
||||
std::sort(array.begin(), array.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
|
||||
for (const auto & point_weight : array)
|
||||
{
|
||||
current_intersections += point_weight.second;
|
||||
if (current_intersections > max_intersections)
|
||||
{
|
||||
max_intersections = current_intersections;
|
||||
position_of_max_intersections = point_weight.first;
|
||||
}
|
||||
}
|
||||
|
||||
if (kind == AggregateFunctionIntersectionsKind::Count)
|
||||
{
|
||||
auto & result_column = static_cast<ColumnUInt64 &>(to).getData();
|
||||
result_column.push_back(max_intersections);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto & result_column = static_cast<ColumnVector<PointType> &>(to).getData();
|
||||
result_column.push_back(position_of_max_intersections);
|
||||
}
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override
|
||||
{
|
||||
return __FILE__;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -37,7 +37,11 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con
|
||||
values_types.push_back(array_type->getNestedType());
|
||||
}
|
||||
|
||||
return AggregateFunctionPtr(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
|
||||
AggregateFunctionPtr res(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
|
||||
if (!res)
|
||||
throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,6 +33,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
{
|
||||
assertNoParameters(name, params);
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -51,23 +55,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data>>();
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
|
||||
template <template <typename> class Data, typename DataForVariadic>
|
||||
@ -75,6 +74,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
{
|
||||
assertNoParameters(name, params);
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -93,23 +96,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data<DataTypeUUID::FieldType>>>();
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -122,8 +122,8 @@ struct AggregateFunctionUniqExactData<String>
|
||||
static String getName() { return "uniqExact"; }
|
||||
};
|
||||
|
||||
template <typename T, HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
{
|
||||
using Key = UInt32;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
@ -135,14 +135,15 @@ struct BaseUniqCombinedData
|
||||
TrivialHash,
|
||||
UInt32,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
template <HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData<String, mode>
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedData<String>
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
@ -154,44 +155,14 @@ struct BaseUniqCombinedData<String, mode>
|
||||
TrivialHash,
|
||||
UInt64,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
/// Aggregate functions uniqCombinedRaw, uniqCombinedLinearCounting, and uniqCombinedBiasCorrected
|
||||
/// are intended for development of new versions of the uniqCombined function.
|
||||
/// Users should only use uniqCombined.
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedRawData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::Raw>
|
||||
{
|
||||
static String getName() { return "uniqCombinedRaw"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedLinearCountingData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::LinearCounting>
|
||||
{
|
||||
static String getName() { return "uniqCombinedLinearCounting"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedBiasCorrectedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::BiasCorrected>
|
||||
{
|
||||
static String getName() { return "uniqCombinedBiasCorrected"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::FullFeatured>
|
||||
{
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
@ -288,10 +259,7 @@ struct OneAdder
|
||||
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedRawData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedLinearCountingData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedBiasCorrectedData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
|
||||
{
|
||||
if constexpr (!std::is_same_v<T, String>)
|
||||
{
|
||||
@ -371,7 +339,7 @@ public:
|
||||
|
||||
/** For multiple arguments. To compute, hashes them.
|
||||
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
|
||||
* But (for the possibility of effective implementation), you can not pass several arguments, among which there are tuples.
|
||||
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
|
||||
*/
|
||||
template <typename Data, bool argument_is_tuple>
|
||||
class AggregateFunctionUniqVariadic final : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniqVariadic<Data, argument_is_tuple>>
|
||||
|
||||
@ -42,6 +42,10 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
|
||||
threshold = threshold_param;
|
||||
}
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -60,22 +64,18 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<true>>(argument_types, threshold);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniqUpTo<DataTypeUUID::FieldType>>(threshold);
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <Core/Types.h>
|
||||
@ -32,7 +33,9 @@ struct QuantileExact
|
||||
|
||||
void add(const Value & x)
|
||||
{
|
||||
array.push_back(x);
|
||||
/// We must skip NaNs as they are not compatible with comparison sorting.
|
||||
if (!isNaN(x))
|
||||
array.push_back(x);
|
||||
}
|
||||
|
||||
template <typename Weight>
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -33,12 +34,15 @@ struct QuantileExactWeighted
|
||||
|
||||
void add(const Value & x)
|
||||
{
|
||||
++map[x];
|
||||
/// We must skip NaNs as they are not compatible with comparison sorting.
|
||||
if (!isNaN(x))
|
||||
++map[x];
|
||||
}
|
||||
|
||||
void add(const Value & x, const Weight & weight)
|
||||
{
|
||||
map[x] += weight;
|
||||
if (!isNaN(x))
|
||||
map[x] += weight;
|
||||
}
|
||||
|
||||
void merge(const QuantileExactWeighted & rhs)
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -12,6 +13,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int PARAMETER_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
|
||||
@ -55,6 +57,10 @@ struct QuantileLevels
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
levels[i] = applyVisitor(FieldVisitorConvertToNumber<Float64>(), params[i]);
|
||||
|
||||
if (isNaN(levels[i]) || levels[i] < 0 || levels[i] > 1)
|
||||
throw Exception("Quantile level is out of range [0..1]", ErrorCodes::PARAMETER_OUT_OF_BOUND);
|
||||
|
||||
permutation[i] = i;
|
||||
}
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <Poco/Exception.h>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
@ -67,6 +68,9 @@ public:
|
||||
|
||||
void insert(const T & v)
|
||||
{
|
||||
if (isNaN(v))
|
||||
return;
|
||||
|
||||
sorted = false;
|
||||
++total_values;
|
||||
if (samples.size() < sample_count)
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <Poco/Exception.h>
|
||||
|
||||
|
||||
@ -66,6 +67,9 @@ public:
|
||||
|
||||
void insert(const T & v, const UInt64 determinator)
|
||||
{
|
||||
if (isNaN(v))
|
||||
return;
|
||||
|
||||
const UInt32 hash = intHash64(determinator);
|
||||
if (!good(hash))
|
||||
return;
|
||||
|
||||
@ -65,7 +65,7 @@ void getAggregateFunctionNameAndParametersArray(
|
||||
ParserExpressionList params_parser(false);
|
||||
ASTPtr args_ast = parseQuery(params_parser,
|
||||
parameters_str.data(), parameters_str.data() + parameters_str.size(),
|
||||
"parameters of aggregate function in " + error_context);
|
||||
"parameters of aggregate function in " + error_context, 0);
|
||||
|
||||
ASTExpressionList & args_list = typeid_cast<ASTExpressionList &>(*args_ast);
|
||||
if (args_list.children.empty())
|
||||
|
||||
@ -23,6 +23,7 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory &);
|
||||
|
||||
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
|
||||
void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
|
||||
@ -53,6 +54,7 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionUniqUpTo(factory);
|
||||
registerAggregateFunctionTopK(factory);
|
||||
registerAggregateFunctionsBitwise(factory);
|
||||
registerAggregateFunctionsMaxIntersections(factory);
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT
|
||||
void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
|
||||
{
|
||||
if (table.storage)
|
||||
for (const auto & name : table.storage->getColumnNamesList())
|
||||
for (const auto & name : table.storage->getColumns().getNamesOfPhysical())
|
||||
res.emplace_back(createASTIdentifierForColumnInTable(name, table));
|
||||
else
|
||||
for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
|
||||
@ -315,7 +315,7 @@ void processIdentifier(
|
||||
}
|
||||
else if (table->storage)
|
||||
{
|
||||
info.data_type = table->storage->getDataTypeByName(column_name);
|
||||
info.data_type = table->storage->getColumn(column_name).type;
|
||||
}
|
||||
else
|
||||
throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
|
||||
{
|
||||
writeString(table.second->getName(), out);
|
||||
writeCString("\n\n", out);
|
||||
writeString(table.second->getColumnsList().toString(), out);
|
||||
writeString(table.second->getColumns().getAllPhysical().toString(), out);
|
||||
writeCString("\n", out);
|
||||
}
|
||||
}
|
||||
|
||||
@ -30,7 +30,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
Context context = Context::createGlobal();
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
AnalyzeLambdas analyzer;
|
||||
analyzer.process(ast);
|
||||
|
||||
@ -26,7 +26,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
Context context = Context::createGlobal();
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
CollectAliases collector;
|
||||
collector.process(ast);
|
||||
|
||||
@ -27,7 +27,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
Context context = Context::createGlobal();
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
Context context = Context::createGlobal();
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
TranslatePositionalArguments translator;
|
||||
translator.process(ast);
|
||||
|
||||
@ -38,7 +38,7 @@ try
|
||||
readStringUntilEOF(query, in);
|
||||
|
||||
ParserSelectQuery parser;
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query");
|
||||
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "query", 0);
|
||||
|
||||
Context context = Context::createGlobal();
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <DataStreams/NativeBlockInputStream.h>
|
||||
#include <DataStreams/NativeBlockOutputStream.h>
|
||||
#include <Client/Connection.h>
|
||||
#include <Client/TimeoutSetter.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/NetException.h>
|
||||
@ -50,14 +51,15 @@ void Connection::connect()
|
||||
if (connected)
|
||||
disconnect();
|
||||
|
||||
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user);
|
||||
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
|
||||
<< (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
|
||||
|
||||
if (static_cast<bool>(encryption))
|
||||
if (static_cast<bool>(secure))
|
||||
{
|
||||
#if Poco_NetSSL_FOUND
|
||||
socket = std::make_unique<Poco::Net::SecureStreamSocket>();
|
||||
#else
|
||||
throw Exception{"tcp_ssl protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
throw Exception{"tcp_secure protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
}
|
||||
else
|
||||
@ -148,6 +150,10 @@ void Connection::receiveHello()
|
||||
{
|
||||
readStringBinary(server_timezone, *in);
|
||||
}
|
||||
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
|
||||
{
|
||||
readStringBinary(server_display_name, *in);
|
||||
}
|
||||
}
|
||||
else if (packet_type == Protocol::Server::Exception)
|
||||
receiveException()->rethrow();
|
||||
@ -203,6 +209,14 @@ const String & Connection::getServerTimezone()
|
||||
return server_timezone;
|
||||
}
|
||||
|
||||
const String & Connection::getServerDisplayName()
|
||||
{
|
||||
if (!connected)
|
||||
connect();
|
||||
|
||||
return server_display_name;
|
||||
}
|
||||
|
||||
void Connection::forceConnected()
|
||||
{
|
||||
if (!connected)
|
||||
@ -216,37 +230,11 @@ void Connection::forceConnected()
|
||||
}
|
||||
}
|
||||
|
||||
struct TimeoutSetter
|
||||
{
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
|
||||
: socket(socket_), timeout(timeout_)
|
||||
{
|
||||
old_send_timeout = socket.getSendTimeout();
|
||||
old_receive_timeout = socket.getReceiveTimeout();
|
||||
|
||||
if (old_send_timeout > timeout)
|
||||
socket.setSendTimeout(timeout);
|
||||
if (old_receive_timeout > timeout)
|
||||
socket.setReceiveTimeout(timeout);
|
||||
}
|
||||
|
||||
~TimeoutSetter()
|
||||
{
|
||||
socket.setSendTimeout(old_send_timeout);
|
||||
socket.setReceiveTimeout(old_receive_timeout);
|
||||
}
|
||||
|
||||
Poco::Net::StreamSocket & socket;
|
||||
Poco::Timespan timeout;
|
||||
Poco::Timespan old_send_timeout;
|
||||
Poco::Timespan old_receive_timeout;
|
||||
};
|
||||
|
||||
bool Connection::ping()
|
||||
{
|
||||
// LOG_TRACE(log_wrapper.get(), "Ping");
|
||||
|
||||
TimeoutSetter timeout_setter(*socket, sync_request_timeout);
|
||||
TimeoutSetter timeout_setter(*socket, sync_request_timeout, true);
|
||||
try
|
||||
{
|
||||
UInt64 pong = 0;
|
||||
@ -286,7 +274,7 @@ TablesStatusResponse Connection::getTablesStatus(const TablesStatusRequest & req
|
||||
if (!connected)
|
||||
connect();
|
||||
|
||||
TimeoutSetter timeout_setter(*socket, sync_request_timeout);
|
||||
TimeoutSetter timeout_setter(*socket, sync_request_timeout, true);
|
||||
|
||||
writeVarUInt(Protocol::Client::TablesStatusRequest, *out);
|
||||
request.write(*out, server_revision);
|
||||
|
||||
@ -58,14 +58,14 @@ public:
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
|
||||
:
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(host, port),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
encryption(encryption_),
|
||||
secure(secure_),
|
||||
timeouts(timeouts_),
|
||||
sync_request_timeout(sync_request_timeout_),
|
||||
log_wrapper(*this)
|
||||
@ -84,7 +84,7 @@ public:
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
|
||||
:
|
||||
host(host_), port(port_),
|
||||
@ -93,7 +93,7 @@ public:
|
||||
resolved_address(resolved_address_),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
encryption(encryption_),
|
||||
secure(secure_),
|
||||
timeouts(timeouts_),
|
||||
sync_request_timeout(sync_request_timeout_),
|
||||
log_wrapper(*this)
|
||||
@ -134,6 +134,7 @@ public:
|
||||
void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
|
||||
|
||||
const String & getServerTimezone();
|
||||
const String & getServerDisplayName();
|
||||
|
||||
/// For log and exception messages.
|
||||
const String & getDescription() const;
|
||||
@ -213,6 +214,7 @@ private:
|
||||
UInt64 server_version_minor = 0;
|
||||
UInt64 server_revision = 0;
|
||||
String server_timezone;
|
||||
String server_display_name;
|
||||
|
||||
std::unique_ptr<Poco::Net::StreamSocket> socket;
|
||||
std::shared_ptr<ReadBuffer> in;
|
||||
@ -220,7 +222,7 @@ private:
|
||||
|
||||
String query_id;
|
||||
Protocol::Compression compression; /// Enable data compression for communication.
|
||||
Protocol::Encryption encryption; /// Enable data encryption for communication.
|
||||
Protocol::Secure secure; /// Enable data encryption for communication.
|
||||
|
||||
/// What compression settings to use while sending data for INSERT queries and external tables.
|
||||
CompressionSettings compression_settings;
|
||||
|
||||
@ -51,11 +51,12 @@ public:
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable)
|
||||
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(host_, port_),
|
||||
client_name(client_name_), compression(compression_), encryption(encryption_),
|
||||
client_name(client_name_), compression(compression_),
|
||||
secure{secure_},
|
||||
timeouts(timeouts)
|
||||
{
|
||||
}
|
||||
@ -67,11 +68,12 @@ public:
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable)
|
||||
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(resolved_address_),
|
||||
client_name(client_name_), compression(compression_), encryption(encryption_),
|
||||
client_name(client_name_), compression(compression_),
|
||||
secure{secure_},
|
||||
timeouts(timeouts)
|
||||
{
|
||||
}
|
||||
@ -102,7 +104,7 @@ protected:
|
||||
return std::make_shared<Connection>(
|
||||
host, port, resolved_address,
|
||||
default_database, user, password, timeouts,
|
||||
client_name, compression, encryption);
|
||||
client_name, compression, secure);
|
||||
}
|
||||
|
||||
private:
|
||||
@ -119,7 +121,7 @@ private:
|
||||
|
||||
String client_name;
|
||||
Protocol::Compression compression; /// Whether to compress data when interacting with the server.
|
||||
Protocol::Encryption encryption; /// Whether to encrypt data when interacting with the server.
|
||||
Protocol::Secure secure; /// Whether to encrypt data when interacting with the server.
|
||||
|
||||
ConnectionTimeouts timeouts;
|
||||
};
|
||||
|
||||
@ -280,8 +280,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
|
||||
read_list.push_back(*connection->socket);
|
||||
}
|
||||
|
||||
/// If no data was found, then we check if there are any connections
|
||||
/// ready for reading.
|
||||
/// If no data was found, then we check if there are any connections ready for reading.
|
||||
if (read_list.empty())
|
||||
{
|
||||
Poco::Net::Socket::SocketList write_list;
|
||||
@ -300,6 +299,9 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
|
||||
throw Exception("Timeout exceeded while reading from " + dumpAddressesUnlocked(), ErrorCodes::TIMEOUT_EXCEEDED);
|
||||
}
|
||||
|
||||
/// TODO Absolutely wrong code: read_list could be empty; rand() is not thread safe and has low quality; motivation of rand is unclear.
|
||||
/// This code path is disabled by default.
|
||||
|
||||
auto & socket = read_list[rand() % read_list.size()];
|
||||
if (fd_to_replica_state_idx.empty())
|
||||
{
|
||||
|
||||
47
dbms/src/Client/TimeoutSetter.h
Normal file
47
dbms/src/Client/TimeoutSetter.h
Normal file
@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Timespan.h>
|
||||
#include <Poco/Net/StreamSocket.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Temporarily overrides socket send/recieve timeouts and reset them back into destructor
|
||||
/// If "limit_max_timeout" is true, timeouts could be only decreased (maxed by previous value).
|
||||
struct TimeoutSetter
|
||||
{
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & send_timeout_, const Poco::Timespan & recieve_timeout_,
|
||||
bool limit_max_timeout = false)
|
||||
: socket(socket_), send_timeout(send_timeout_), recieve_timeout(recieve_timeout_)
|
||||
{
|
||||
old_send_timeout = socket.getSendTimeout();
|
||||
old_receive_timeout = socket.getReceiveTimeout();
|
||||
|
||||
if (!limit_max_timeout || old_send_timeout > send_timeout)
|
||||
socket.setSendTimeout(send_timeout);
|
||||
|
||||
if (!limit_max_timeout || old_receive_timeout > recieve_timeout)
|
||||
socket.setReceiveTimeout(recieve_timeout);
|
||||
}
|
||||
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_, bool limit_max_timeout = false)
|
||||
: TimeoutSetter(socket_, timeout_, timeout_, limit_max_timeout) {}
|
||||
|
||||
~TimeoutSetter()
|
||||
{
|
||||
socket.setSendTimeout(old_send_timeout);
|
||||
socket.setReceiveTimeout(old_receive_timeout);
|
||||
}
|
||||
|
||||
Poco::Net::StreamSocket & socket;
|
||||
|
||||
Poco::Timespan send_timeout;
|
||||
Poco::Timespan recieve_timeout;
|
||||
|
||||
Poco::Timespan old_send_timeout;
|
||||
Poco::Timespan old_receive_timeout;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = getData().size();
|
||||
if (size != filter.size())
|
||||
@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = getData().size();
|
||||
|
||||
@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n)
|
||||
data.resize_assume_reserved(new_size);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
|
||||
ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != offsets.size())
|
||||
|
||||
@ -152,11 +152,11 @@ public:
|
||||
|
||||
void popBack(size_t n) override;
|
||||
|
||||
MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||
|
||||
|
||||
@ -31,10 +31,10 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
|
||||
: data(nested_column), offsets(offsets_column)
|
||||
ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
|
||||
: data(std::move(nested_column)), offsets(std::move(offsets_column))
|
||||
{
|
||||
if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
|
||||
if (!typeid_cast<const ColumnOffsets *>(offsets.get()))
|
||||
throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
/** NOTE
|
||||
@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs
|
||||
*/
|
||||
}
|
||||
|
||||
ColumnArray::ColumnArray(const ColumnPtr & nested_column)
|
||||
: data(nested_column)
|
||||
ColumnArray::ColumnArray(MutableColumnPtr && nested_column)
|
||||
: data(std::move(nested_column))
|
||||
{
|
||||
if (!data->empty())
|
||||
throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN);
|
||||
@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const
|
||||
ColumnPtr ColumnArray::convertToFullColumnIfConst() const
|
||||
{
|
||||
ColumnPtr new_data;
|
||||
|
||||
@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return filterNumber<UInt8>(filt, result_size_hint);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return filterNumber<UInt16>(filt, result_size_hint);
|
||||
@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t col_size = getOffsets().size();
|
||||
if (col_size != filt.size())
|
||||
@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = getOffsets().size();
|
||||
if (size != filt.size())
|
||||
@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result
|
||||
filtered_offsets);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
|
||||
|
||||
Columns temporary_arrays(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint);
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.filter(filt, result_size_hint);
|
||||
|
||||
Columns tuple_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = getOffsets().size();
|
||||
|
||||
@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
{
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return replicateNumber<UInt8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return replicateNumber<UInt16>(replicate_offsets);
|
||||
@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
|
||||
{
|
||||
const ColumnNullable & nullable = static_cast<const ColumnNullable &>(*data);
|
||||
|
||||
/// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result.
|
||||
/// NOTE Offsets are calculated twice and it is redundant.
|
||||
|
||||
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
|
||||
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
|
||||
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
|
||||
return ColumnArray::create(
|
||||
ColumnNullable::create(
|
||||
static_cast<ColumnArray &>(*array_of_nested).getDataPtr(),
|
||||
static_cast<ColumnArray &>(*array_of_null_map).getDataPtr()),
|
||||
static_cast<ColumnArray &>(*array_of_nested).getOffsetsPtr());
|
||||
static_cast<const ColumnArray &>(*array_of_nested).getDataPtr(),
|
||||
static_cast<const ColumnArray &>(*array_of_null_map).getDataPtr()),
|
||||
static_cast<const ColumnArray &>(*array_of_nested).getOffsetsPtr());
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
|
||||
{
|
||||
const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
|
||||
|
||||
@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets)
|
||||
|
||||
Columns temporary_arrays(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets);
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
|
||||
Columns tuple_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
|
||||
@ -24,14 +24,32 @@ private:
|
||||
friend class COWPtrHelper<IColumn, ColumnArray>;
|
||||
|
||||
/** Create an array column with specified values and offsets. */
|
||||
ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column);
|
||||
ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
|
||||
|
||||
/** Create an empty column of arrays with the type of values as in the column `nested_column` */
|
||||
ColumnArray(const ColumnPtr & nested_column);
|
||||
explicit ColumnArray(MutableColumnPtr && nested_column);
|
||||
|
||||
ColumnArray(const ColumnArray &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnArray>;
|
||||
|
||||
static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
|
||||
{
|
||||
return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
|
||||
}
|
||||
|
||||
static Ptr create(const ColumnPtr & nested_column)
|
||||
{
|
||||
return ColumnArray::create(nested_column->assumeMutable());
|
||||
}
|
||||
|
||||
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
|
||||
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
|
||||
|
||||
/** On the index i there is an offset to the beginning of the i + 1 -th element. */
|
||||
using ColumnOffsets = ColumnVector<Offset>;
|
||||
|
||||
@ -51,15 +69,15 @@ public:
|
||||
void insertFrom(const IColumn & src_, size_t n) override;
|
||||
void insertDefault() override;
|
||||
void popBack(size_t n) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
|
||||
void reserve(size_t n) override;
|
||||
size_t byteSize() const override;
|
||||
size_t allocatedBytes() const override;
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
MutableColumnPtr convertToFullColumnIfConst() const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr convertToFullColumnIfConst() const override;
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
bool hasEqualOffsets(const ColumnArray & other) const;
|
||||
@ -110,33 +128,33 @@ private:
|
||||
|
||||
/// Multiply values if the nested column is ColumnVector<T>.
|
||||
template <typename T>
|
||||
MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
|
||||
|
||||
/// Multiply the values if the nested column is ColumnString. The code is too complicated.
|
||||
MutableColumnPtr replicateString(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateString(const Offsets & replicate_offsets) const;
|
||||
|
||||
/** Non-constant arrays of constant values are quite rare.
|
||||
* Most functions can not work with them, and does not create such columns as a result.
|
||||
* An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions.
|
||||
* Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst).
|
||||
*/
|
||||
MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateConst(const Offsets & replicate_offsets) const;
|
||||
|
||||
/** The following is done by simply replicating of nested columns.
|
||||
*/
|
||||
MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
|
||||
MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
|
||||
MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
|
||||
|
||||
|
||||
/// Specializations for the filter function.
|
||||
template <typename T>
|
||||
MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
|
||||
|
||||
MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s)
|
||||
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::convertToFullColumn() const
|
||||
ColumnPtr ColumnConst::convertToFullColumn() const
|
||||
{
|
||||
return data->replicate(Offsets(1, s));
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
|
||||
ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
|
||||
{
|
||||
if (s != filt.size())
|
||||
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")",
|
||||
@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_
|
||||
return ColumnConst::create(data, countBytesInFilter(filt));
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
{
|
||||
if (s != offsets.size())
|
||||
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")",
|
||||
@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
return ColumnConst::create(data, replicated_size);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
if (limit == 0)
|
||||
limit = s;
|
||||
|
||||
@ -29,9 +29,9 @@ private:
|
||||
ColumnConst(const ColumnConst & src) = default;
|
||||
|
||||
public:
|
||||
MutableColumnPtr convertToFullColumn() const;
|
||||
ColumnPtr convertToFullColumn() const;
|
||||
|
||||
MutableColumnPtr convertToFullColumnIfConst() const override
|
||||
ColumnPtr convertToFullColumnIfConst() const override
|
||||
{
|
||||
return convertToFullColumn();
|
||||
}
|
||||
@ -145,9 +145,9 @@ public:
|
||||
data->updateHashWithValue(0, hash);
|
||||
}
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
|
||||
|
||||
size_t byteSize() const override
|
||||
|
||||
@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
|
||||
memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != filt.size())
|
||||
@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
|
||||
@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != offsets.size())
|
||||
|
||||
@ -104,11 +104,11 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
|
||||
{
|
||||
|
||||
@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
|
||||
return ColumnFunction::create(size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
{
|
||||
if (size_ != offsets.size())
|
||||
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column ("
|
||||
@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
return ColumnFunction::create(replicated_size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
{
|
||||
ColumnsWithTypeAndName capture = captured_columns;
|
||||
for (auto & column : capture)
|
||||
@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
return ColumnFunction::create(length, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (size_ != filt.size())
|
||||
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column ("
|
||||
@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size
|
||||
return ColumnFunction::create(filtered_size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
if (limit == 0)
|
||||
limit = size_;
|
||||
|
||||
@ -29,10 +29,10 @@ public:
|
||||
|
||||
size_t size() const override { return size_; }
|
||||
|
||||
MutableColumnPtr cut(size_t start, size_t length) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr cut(size_t start, size_t length) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
void insertDefault() override;
|
||||
void popBack(size_t n) override;
|
||||
std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex num_columns,
|
||||
|
||||
@ -18,8 +18,8 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
|
||||
: nested_column{nested_column_}, null_map{null_map_}
|
||||
ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_)
|
||||
: nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
|
||||
{
|
||||
/// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
|
||||
if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
|
||||
@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
|
||||
|
||||
MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
|
||||
{
|
||||
ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
|
||||
MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
|
||||
auto new_null_map = ColumnUInt8::create();
|
||||
|
||||
if (new_size > 0)
|
||||
@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
|
||||
memset(&new_null_map->getData()[count], 1, new_size - count);
|
||||
}
|
||||
|
||||
return ColumnNullable::create(new_nested_col, std::move(new_null_map));
|
||||
return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map));
|
||||
}
|
||||
|
||||
|
||||
@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n)
|
||||
getNullMapColumn().popBack(n);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint);
|
||||
ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint);
|
||||
return ColumnNullable::create(filtered_data, filtered_null_map);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
ColumnPtr permuted_data = getNestedColumn().permute(perm, limit);
|
||||
ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit);
|
||||
@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
|
||||
{
|
||||
ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
|
||||
ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);
|
||||
|
||||
@ -23,10 +23,22 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
|
||||
private:
|
||||
friend class COWPtrHelper<IColumn, ColumnNullable>;
|
||||
|
||||
ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_);
|
||||
ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
|
||||
ColumnNullable(const ColumnNullable &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnNullable>;
|
||||
static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
|
||||
{
|
||||
return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
|
||||
}
|
||||
|
||||
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
|
||||
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
|
||||
|
||||
const char * getFamilyName() const override { return "Nullable"; }
|
||||
std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
|
||||
MutableColumnPtr cloneResized(size_t size) const override;
|
||||
@ -50,14 +62,14 @@ public:
|
||||
}
|
||||
|
||||
void popBack(size_t n) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
|
||||
void reserve(size_t n) override;
|
||||
size_t byteSize() const override;
|
||||
size_t allocatedBytes() const override;
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
void updateHashWithValue(size_t n, SipHash & hash) const override;
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
|
||||
@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (offsets.size() == 0)
|
||||
return ColumnString::create();
|
||||
@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = offsets.size();
|
||||
|
||||
@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
|
||||
@ -89,6 +89,12 @@ public:
|
||||
return StringRef(&chars[offsetAt(n)], sizeAt(n));
|
||||
}
|
||||
|
||||
/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
|
||||
#endif
|
||||
|
||||
void insert(const Field & x) override
|
||||
{
|
||||
const String & s = DB::get<const String &>(x);
|
||||
@ -101,7 +107,11 @@ public:
|
||||
offsets.push_back(new_size);
|
||||
}
|
||||
|
||||
void insertFrom(const IColumn & src_, size_t n) override
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
void insertFrom(const IColumn & src_, size_t n) override
|
||||
{
|
||||
const ColumnString & src = static_cast<const ColumnString &>(src_);
|
||||
|
||||
@ -206,9 +216,9 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
void insertDefault() override
|
||||
{
|
||||
@ -239,7 +249,7 @@ public:
|
||||
/// Sorting with respect of collation.
|
||||
void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
|
||||
{
|
||||
|
||||
@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const
|
||||
return res.str();
|
||||
}
|
||||
|
||||
ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns)
|
||||
ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
|
||||
{
|
||||
columns.reserve(mutable_columns.size());
|
||||
for (auto & column : mutable_columns)
|
||||
{
|
||||
if (column->isColumnConst())
|
||||
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
|
||||
|
||||
columns.push_back(std::move(column));
|
||||
}
|
||||
}
|
||||
|
||||
ColumnTuple::Ptr ColumnTuple::create(const Columns & columns)
|
||||
{
|
||||
for (const auto & column : columns)
|
||||
if (column->isColumnConst())
|
||||
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
|
||||
|
||||
auto column_tuple = ColumnTuple::create(MutableColumns());
|
||||
column_tuple->columns = columns;
|
||||
|
||||
return std::move(column_tuple);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::cloneEmpty() const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
MutableColumns new_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
new_columns[i] = columns[i]->cloneEmpty();
|
||||
|
||||
return ColumnTuple::create(new_columns);
|
||||
return ColumnTuple::create(std::move(new_columns));
|
||||
}
|
||||
|
||||
Field ColumnTuple::operator[](size_t n) const
|
||||
@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng
|
||||
start, length);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi
|
||||
return ColumnTuple::create(new_columns);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co
|
||||
return ColumnTuple::create(new_columns);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se
|
||||
|
||||
for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
|
||||
{
|
||||
Columns new_columns(tuple_size);
|
||||
MutableColumns new_columns(tuple_size);
|
||||
for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx)
|
||||
new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
|
||||
res[scattered_idx] = ColumnTuple::create(new_columns);
|
||||
res[scattered_idx] = ColumnTuple::create(std::move(new_columns));
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@ -22,10 +22,19 @@ private:
|
||||
template <bool positive>
|
||||
struct Less;
|
||||
|
||||
ColumnTuple(const Columns & columns);
|
||||
explicit ColumnTuple(MutableColumns && columns);
|
||||
ColumnTuple(const ColumnTuple &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnTuple>;
|
||||
static Ptr create(const Columns & columns);
|
||||
|
||||
template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
|
||||
static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
|
||||
|
||||
std::string getName() const override;
|
||||
const char * getFamilyName() const override { return "Tuple"; }
|
||||
|
||||
@ -49,9 +58,9 @@ public:
|
||||
const char * deserializeAndInsertFromArena(const char * pos) override;
|
||||
void updateHashWithValue(size_t n, SipHash & hash) const override;
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||
void gather(ColumnGathererStream & gatherer_stream) override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
|
||||
|
||||
@ -146,7 +146,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != filt.size())
|
||||
@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t r
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
|
||||
@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, siz
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
ColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != offsets.size())
|
||||
|
||||
@ -243,11 +243,11 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
|
||||
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
|
||||
@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column)
|
||||
|
||||
if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(&column))
|
||||
{
|
||||
MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate();
|
||||
ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
|
||||
MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate();
|
||||
|
||||
ColumnUInt8 * concrete_column = typeid_cast<ColumnUInt8 *>(mutable_holder.get());
|
||||
if (!concrete_column)
|
||||
|
||||
@ -45,7 +45,7 @@ public:
|
||||
/** If column isn't constant, returns nullptr (or itself).
|
||||
* If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
|
||||
*/
|
||||
virtual MutablePtr convertToFullColumnIfConst() const { return {}; }
|
||||
virtual Ptr convertToFullColumnIfConst() const { return {}; }
|
||||
|
||||
/// Creates empty column with the same type.
|
||||
virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
|
||||
@ -104,11 +104,11 @@ public:
|
||||
|
||||
/// Removes all elements outside of specified range.
|
||||
/// Is used in LIMIT operation, for example.
|
||||
virtual MutablePtr cut(size_t start, size_t length) const
|
||||
virtual Ptr cut(size_t start, size_t length) const
|
||||
{
|
||||
MutablePtr res = cloneEmpty();
|
||||
res->insertRangeFrom(*this, start, length);
|
||||
return res;
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
/// Appends new value at the end of column (column's size is increased by 1).
|
||||
@ -171,12 +171,12 @@ public:
|
||||
* otherwise (i.e. < 0), makes reserve() using size of source column.
|
||||
*/
|
||||
using Filter = PaddedPODArray<UInt8>;
|
||||
virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
|
||||
virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
|
||||
|
||||
/// Permutes elements using specified permutation. Is used in sortings.
|
||||
/// limit - if it isn't 0, puts only first limit elements in the result.
|
||||
using Permutation = PaddedPODArray<size_t>;
|
||||
virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0;
|
||||
virtual Ptr permute(const Permutation & perm, size_t limit) const = 0;
|
||||
|
||||
/** Compares (*this)[n] and rhs[m].
|
||||
* Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively.
|
||||
@ -205,7 +205,7 @@ public:
|
||||
*/
|
||||
using Offset = UInt64;
|
||||
using Offsets = PaddedPODArray<Offset>;
|
||||
virtual MutablePtr replicate(const Offsets & offsets) const = 0;
|
||||
virtual Ptr replicate(const Offsets & offsets) const = 0;
|
||||
|
||||
/** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
|
||||
* Selector must contain values from 0 to num_columns - 1.
|
||||
@ -247,10 +247,10 @@ public:
|
||||
virtual void forEachSubcolumn(ColumnCallback) {}
|
||||
|
||||
|
||||
MutablePtr mutate() const
|
||||
MutablePtr mutate() const &&
|
||||
{
|
||||
MutablePtr res = COWPtr<IColumn>::mutate();
|
||||
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); });
|
||||
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); });
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -360,4 +360,16 @@ using MutableColumns = std::vector<MutableColumnPtr>;
|
||||
using ColumnRawPtrs = std::vector<const IColumn *>;
|
||||
//using MutableColumnRawPtrs = std::vector<IColumn *>;
|
||||
|
||||
template <typename ... Args>
|
||||
struct IsMutableColumns;
|
||||
|
||||
template <typename Arg, typename ... Args>
|
||||
struct IsMutableColumns<Arg, Args ...>
|
||||
{
|
||||
static const bool value = std::is_assignable<MutableColumnPtr &&, Arg>::value && IsMutableColumns<Args ...>::value;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct IsMutableColumns<> { static const bool value = true; };
|
||||
|
||||
}
|
||||
|
||||
@ -74,12 +74,12 @@ public:
|
||||
s += length;
|
||||
}
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
|
||||
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
|
||||
{
|
||||
return cloneDummy(countBytesInFilter(filt));
|
||||
}
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override
|
||||
{
|
||||
if (s != perm.size())
|
||||
throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
@ -94,7 +94,7 @@ public:
|
||||
res[i] = i;
|
||||
}
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override
|
||||
ColumnPtr replicate(const Offsets & offsets) const override
|
||||
{
|
||||
if (s != offsets.size())
|
||||
throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#if !(defined(__FreeBSD__) || defined(__APPLE__))
|
||||
#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
@ -68,10 +68,9 @@
|
||||
* of this shared state.
|
||||
*
|
||||
* Caveats:
|
||||
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere
|
||||
* and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified.
|
||||
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere.
|
||||
* - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better
|
||||
* to use std::unique_ptr for it, but see above.
|
||||
* to use std::unique_ptr for it somehow.
|
||||
*/
|
||||
template <typename Derived>
|
||||
class COWPtr : public boost::intrusive_ref_counter<Derived>
|
||||
@ -80,12 +79,22 @@ private:
|
||||
Derived * derived() { return static_cast<Derived *>(this); }
|
||||
const Derived * derived() const { return static_cast<const Derived *>(this); }
|
||||
|
||||
template <typename T>
|
||||
class IntrusivePtr : public boost::intrusive_ptr<T>
|
||||
{
|
||||
public:
|
||||
using boost::intrusive_ptr<T>::intrusive_ptr;
|
||||
|
||||
T & operator*() const & { return boost::intrusive_ptr<T>::operator*(); }
|
||||
T && operator*() const && { return const_cast<typename std::remove_const<T>::type &&>(*boost::intrusive_ptr<T>::get()); }
|
||||
};
|
||||
|
||||
protected:
|
||||
template <typename T>
|
||||
class mutable_ptr : public boost::intrusive_ptr<T>
|
||||
class mutable_ptr : public IntrusivePtr<T>
|
||||
{
|
||||
private:
|
||||
using Base = boost::intrusive_ptr<T>;
|
||||
using Base = IntrusivePtr<T>;
|
||||
|
||||
template <typename> friend class COWPtr;
|
||||
template <typename, typename> friend class COWPtrHelper;
|
||||
@ -114,10 +123,10 @@ public:
|
||||
|
||||
protected:
|
||||
template <typename T>
|
||||
class immutable_ptr : public boost::intrusive_ptr<const T>
|
||||
class immutable_ptr : public IntrusivePtr<const T>
|
||||
{
|
||||
private:
|
||||
using Base = boost::intrusive_ptr<const T>;
|
||||
using Base = IntrusivePtr<const T>;
|
||||
|
||||
template <typename> friend class COWPtr;
|
||||
template <typename, typename> friend class COWPtrHelper;
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <Poco/DOM/Comment.h>
|
||||
#include <Poco/Util/XMLConfiguration.h>
|
||||
#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
|
||||
#define PREPROCESSED_SUFFIX "-preprocessed"
|
||||
@ -276,6 +277,8 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
/// Replace the original contents, not add to it.
|
||||
bool replace = attributes->getNamedItem("replace");
|
||||
|
||||
bool included_something = false;
|
||||
|
||||
auto process_include = [&](const Node * include_attr, const std::function<const Node * (const std::string &)> & get_node, const char * error_msg)
|
||||
{
|
||||
std::string name = include_attr->getNodeValue();
|
||||
@ -316,6 +319,8 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
{
|
||||
element->setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
|
||||
}
|
||||
|
||||
included_something = true;
|
||||
}
|
||||
};
|
||||
|
||||
@ -348,11 +353,14 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
}
|
||||
}
|
||||
|
||||
NodeListPtr children = node->childNodes();
|
||||
Node * child = nullptr;
|
||||
for (size_t i = 0; (child = children->item(i)); ++i)
|
||||
if (included_something)
|
||||
doIncludesRecursive(config, include_from, node, zk_node_cache, contributing_zk_paths);
|
||||
else
|
||||
{
|
||||
doIncludesRecursive(config, include_from, child, zk_node_cache, contributing_zk_paths);
|
||||
NodeListPtr children = node->childNodes();
|
||||
Node * child = nullptr;
|
||||
for (size_t i = 0; (child = children->item(i)); ++i)
|
||||
doIncludesRecursive(config, include_from, child, zk_node_cache, contributing_zk_paths);
|
||||
}
|
||||
}
|
||||
|
||||
@ -375,15 +383,17 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
|
||||
Poco::File merge_dir(merge_dir_name);
|
||||
if (!merge_dir.exists() || !merge_dir.isDirectory())
|
||||
continue;
|
||||
|
||||
for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it)
|
||||
{
|
||||
Poco::File & file = *it;
|
||||
if (file.isFile()
|
||||
&& (endsWith(file.path(), ".xml") || endsWith(file.path(), ".conf"))
|
||||
&& !startsWith(file.path(), ".")) // skip temporary files
|
||||
{
|
||||
Poco::Path path(file.path());
|
||||
std::string extension = path.getExtension();
|
||||
std::string base_name = path.getBaseName();
|
||||
|
||||
// Skip non-config and temporary files
|
||||
if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, "."))
|
||||
files.push_back(file.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -67,12 +67,15 @@ void ConfigReloader::run()
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(log, __PRETTY_FUNCTION__);
|
||||
std::this_thread::sleep_for(reload_interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(reload_mutex);
|
||||
|
||||
FilesChangesTracker new_files = getNewFileList();
|
||||
if (force || new_files.isDifferOrNewerThan(files))
|
||||
{
|
||||
|
||||
@ -42,6 +42,9 @@ public:
|
||||
/// Call this method to run the backround thread.
|
||||
void start();
|
||||
|
||||
/// Reload immediately. For SYSTEM RELOAD CONFIG query.
|
||||
void reload() { reloadIfNewer(/* force */ true, /* throw_on_error */ true, /* fallback_to_preprocessed */ false); }
|
||||
|
||||
private:
|
||||
void run();
|
||||
|
||||
@ -74,6 +77,9 @@ private:
|
||||
|
||||
std::atomic<bool> quit{false};
|
||||
std::thread thread;
|
||||
|
||||
/// Locked inside reloadIfNewer.
|
||||
std::mutex reload_mutex;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -28,7 +28,9 @@
|
||||
M(MemoryTrackingForMerges) \
|
||||
M(LeaderElection) \
|
||||
M(EphemeralNode) \
|
||||
M(ZooKeeperSession) \
|
||||
M(ZooKeeperWatch) \
|
||||
M(ZooKeeperRequest) \
|
||||
M(DelayedInserts) \
|
||||
M(ContextLockWait) \
|
||||
M(StorageBufferRows) \
|
||||
|
||||
@ -125,7 +125,6 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_SETTING = 115;
|
||||
extern const int THERE_IS_NO_DEFAULT_VALUE = 116;
|
||||
extern const int INCORRECT_DATA = 117;
|
||||
extern const int TABLE_METADATA_DOESNT_EXIST = 118;
|
||||
extern const int ENGINE_REQUIRED = 119;
|
||||
extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE = 120;
|
||||
extern const int UNKNOWN_SET_DATA_VARIANT = 121;
|
||||
@ -372,6 +371,7 @@ namespace ErrorCodes
|
||||
extern const int QUERY_WAS_CANCELLED = 394;
|
||||
extern const int FUNCTION_THROW_IF_VALUE_IS_NON_ZERO = 395;
|
||||
extern const int TOO_MANY_ROWS_OR_BYTES = 396;
|
||||
extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397;
|
||||
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/demangle.h>
|
||||
#include <common/demangle.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -198,7 +198,7 @@ public:
|
||||
|
||||
/// Create table
|
||||
NamesAndTypesList columns = sample_block.getNamesAndTypesList();
|
||||
StoragePtr storage = StorageMemory::create(data.second, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{});
|
||||
StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
|
||||
storage->startup();
|
||||
context.addExternalTable(data.second, storage);
|
||||
BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
#include <Core/Field.h>
|
||||
#include <Core/AccurateComparison.h>
|
||||
#include <common/DateLUT.h>
|
||||
#include <Common/demangle.h>
|
||||
#include <common/demangle.h>
|
||||
|
||||
|
||||
class SipHash;
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <Common/Macros.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
@ -1,10 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <map>
|
||||
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
namespace Util
|
||||
{
|
||||
class AbstractConfiguration;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -21,10 +29,12 @@ public:
|
||||
*/
|
||||
String expand(const String & s, size_t level = 0) const;
|
||||
|
||||
private:
|
||||
using MacroMap = std::map<String, String>;
|
||||
const MacroMap getMacroMap() const { return macros; }
|
||||
|
||||
private:
|
||||
MacroMap macros;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -56,14 +56,20 @@
|
||||
\
|
||||
M(ZooKeeperInit) \
|
||||
M(ZooKeeperTransactions) \
|
||||
M(ZooKeeperGetChildren) \
|
||||
M(ZooKeeperList) \
|
||||
M(ZooKeeperCreate) \
|
||||
M(ZooKeeperRemove) \
|
||||
M(ZooKeeperExists) \
|
||||
M(ZooKeeperGet) \
|
||||
M(ZooKeeperSet) \
|
||||
M(ZooKeeperMulti) \
|
||||
M(ZooKeeperCheck) \
|
||||
M(ZooKeeperClose) \
|
||||
M(ZooKeeperWatchResponse) \
|
||||
M(ZooKeeperExceptions) \
|
||||
M(ZooKeeperWaitMicroseconds) \
|
||||
M(ZooKeeperBytesSent) \
|
||||
M(ZooKeeperBytesReceived) \
|
||||
\
|
||||
M(DistributedConnectionFailTry) \
|
||||
M(DistributedConnectionMissingTable) \
|
||||
|
||||
@ -1,13 +1,12 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <dlfcn.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <IO/WriteBufferFromVector.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <sstream>
|
||||
|
||||
#include <Common/StackTrace.h>
|
||||
#include <Common/demangle.h>
|
||||
#include <common/demangle.h>
|
||||
|
||||
|
||||
StackTrace::StackTrace()
|
||||
|
||||
@ -27,6 +27,9 @@ namespace ErrorCodes
|
||||
class Throttler
|
||||
{
|
||||
public:
|
||||
Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent = nullptr)
|
||||
: max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent) {}
|
||||
|
||||
Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
|
||||
const std::shared_ptr<Throttler> & parent = nullptr)
|
||||
: max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent) {}
|
||||
@ -76,6 +79,12 @@ public:
|
||||
parent->add(amount);
|
||||
}
|
||||
|
||||
/// Not thread safe
|
||||
void setParent(const std::shared_ptr<Throttler> & parent_)
|
||||
{
|
||||
parent = parent_;
|
||||
}
|
||||
|
||||
void reset()
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
@ -3,6 +3,9 @@
|
||||
#include <Core/Types.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
|
||||
#if __SSE2__
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -49,9 +52,29 @@ inline size_t seqLength(const UInt8 first_octet)
|
||||
inline size_t countCodePoints(const UInt8 * data, size_t size)
|
||||
{
|
||||
size_t res = 0;
|
||||
const auto end = data + size;
|
||||
|
||||
/// TODO SIMD implementation looks quite simple.
|
||||
for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes.
|
||||
#if __SSE2__
|
||||
const auto bytes_sse = sizeof(__m128i);
|
||||
const auto src_end_sse = (data + size) - (size % bytes_sse);
|
||||
|
||||
const auto align_sse = _mm_set1_epi8(0x40);
|
||||
const auto upper_bound = _mm_set1_epi8(0xBF);
|
||||
|
||||
for (; data < src_end_sse; data += bytes_sse)
|
||||
{
|
||||
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
|
||||
|
||||
///Align to zero for the solve two case
|
||||
const auto align_res = _mm_adds_epu8(chars, align_sse);
|
||||
const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
|
||||
|
||||
res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
|
||||
res += (*data <= 0x7F || *data >= 0xC0);
|
||||
|
||||
return res;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/demangle.h>
|
||||
#include <common/demangle.h>
|
||||
#include <Common/TypeList.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
|
||||
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
|
||||
|
||||
add_headers_and_sources(clickhouse_common_zookeeper .)
|
||||
|
||||
@ -29,11 +29,11 @@ public:
|
||||
if (zookeeper->tryGet(path, result_str, &stat))
|
||||
{
|
||||
result = std::stol(result_str) + 1;
|
||||
success = zookeeper->trySet(path, std::to_string(result), stat.version) == ZOK;
|
||||
success = zookeeper->trySet(path, std::to_string(result), stat.version) == ZooKeeperImpl::ZooKeeper::ZOK;
|
||||
}
|
||||
else
|
||||
{
|
||||
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == ZOK;
|
||||
success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == ZooKeeperImpl::ZooKeeper::ZOK;
|
||||
}
|
||||
}
|
||||
while (!success);
|
||||
|
||||
@ -1,21 +1,6 @@
|
||||
#pragma once
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include "Types.h"
|
||||
#include <Common/ProfileEvents.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int KEEPER_EXCEPTION;
|
||||
}
|
||||
}
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event ZooKeeperExceptions;
|
||||
}
|
||||
|
||||
|
||||
namespace zkutil
|
||||
@ -23,83 +8,46 @@ namespace zkutil
|
||||
|
||||
|
||||
/// You should reinitialize ZooKeeper session in case of these errors
|
||||
inline bool isUnrecoverableErrorCode(int32_t zk_return_code)
|
||||
inline bool isHardwareError(int32_t zk_return_code)
|
||||
{
|
||||
return zk_return_code == ZINVALIDSTATE || zk_return_code == ZSESSIONEXPIRED || zk_return_code == ZSESSIONMOVED;
|
||||
}
|
||||
|
||||
/// Errors related with temporary network problems
|
||||
inline bool isTemporaryErrorCode(int32_t zk_return_code)
|
||||
{
|
||||
return zk_return_code == ZCONNECTIONLOSS || zk_return_code == ZOPERATIONTIMEOUT;
|
||||
}
|
||||
|
||||
/// Any error related with network or master election
|
||||
/// In case of these errors you should retry the query or reinitialize ZooKeeper session (see isUnrecoverable())
|
||||
inline bool isHardwareErrorCode(int32_t zk_return_code)
|
||||
{
|
||||
return isUnrecoverableErrorCode(zk_return_code) || isTemporaryErrorCode(zk_return_code);
|
||||
return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT;
|
||||
}
|
||||
|
||||
/// Valid errors sent from server
|
||||
inline bool isUserError(int32_t zk_return_code)
|
||||
{
|
||||
return zk_return_code == ZNONODE
|
||||
|| zk_return_code == ZBADVERSION
|
||||
|| zk_return_code == ZNOCHILDRENFOREPHEMERALS
|
||||
|| zk_return_code == ZNODEEXISTS
|
||||
|| zk_return_code == ZNOTEMPTY;
|
||||
return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS
|
||||
|| zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY;
|
||||
}
|
||||
|
||||
|
||||
class KeeperException : public DB::Exception
|
||||
using KeeperException = ZooKeeperImpl::Exception;
|
||||
|
||||
|
||||
class KeeperMultiException : public KeeperException
|
||||
{
|
||||
private:
|
||||
/// delegate constructor, used to minimize repetition; last parameter used for overload resolution
|
||||
KeeperException(const std::string & msg, const int32_t code, int)
|
||||
: DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code) { incrementEventCounter(); }
|
||||
|
||||
public:
|
||||
explicit KeeperException(const std::string & msg) : KeeperException(msg, ZOK, 0) {}
|
||||
KeeperException(const std::string & msg, const int32_t code)
|
||||
: KeeperException(msg + " (" + zerror(code) + ")", code, 0) {}
|
||||
explicit KeeperException(const int32_t code) : KeeperException(zerror(code), code, 0) {}
|
||||
KeeperException(const int32_t code, const std::string & path)
|
||||
: KeeperException(std::string{zerror(code)} + ", path: " + path, code, 0) {}
|
||||
Requests requests;
|
||||
Responses responses;
|
||||
size_t failed_op_index = 0;
|
||||
|
||||
KeeperException(const KeeperException & exc) : DB::Exception(exc), code(exc.code) { incrementEventCounter(); }
|
||||
std::string getPathForFirstFailedOp() const;
|
||||
|
||||
const char * name() const throw() override { return "zkutil::KeeperException"; }
|
||||
const char * className() const throw() override { return "zkutil::KeeperException"; }
|
||||
KeeperException * clone() const override { return new KeeperException(*this); }
|
||||
/// If it is user error throws KeeperMultiException else throws ordinary KeeperException
|
||||
/// If it is ZOK does nothing
|
||||
static void check(int32_t code, const Requests & requests, const Responses & responses);
|
||||
|
||||
/// You should reinitialize ZooKeeper session in case of these errors
|
||||
bool isUnrecoverable() const
|
||||
{
|
||||
return isUnrecoverableErrorCode(code);
|
||||
}
|
||||
|
||||
/// Errors related with temporary network problems
|
||||
bool isTemporaryError() const
|
||||
{
|
||||
return isTemporaryErrorCode(code);
|
||||
}
|
||||
|
||||
/// Any error related with network or master election
|
||||
/// In case of these errors you should retry the query or reinitialize ZooKeeper session (see isUnrecoverable())
|
||||
bool isHardwareError() const
|
||||
{
|
||||
return isHardwareErrorCode(code);
|
||||
}
|
||||
|
||||
const int32_t code;
|
||||
KeeperMultiException(int32_t code, const Requests & requests, const Responses & responses);
|
||||
|
||||
private:
|
||||
static void incrementEventCounter()
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperExceptions);
|
||||
}
|
||||
|
||||
size_t getFailedOpIndex(int32_t code, const Responses & responses) const;
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
@ -33,8 +33,7 @@ public:
|
||||
*
|
||||
* identifier - if not empty, must uniquely (within same path) identify participant of leader election.
|
||||
* It means that different participants of leader election have different identifiers
|
||||
* and existence of more than one ephemeral node with same identifier indicates an error
|
||||
* (see cleanOldEphemeralNodes).
|
||||
* and existence of more than one ephemeral node with same identifier indicates an error.
|
||||
*/
|
||||
LeaderElection(const std::string & path_, ZooKeeper & zookeeper_, LeadershipHandler handler_, const std::string & identifier_ = "")
|
||||
: path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_)
|
||||
@ -42,10 +41,15 @@ public:
|
||||
createNode();
|
||||
}
|
||||
|
||||
void yield()
|
||||
void shutdown()
|
||||
{
|
||||
releaseNode();
|
||||
createNode();
|
||||
if (shutdown_called)
|
||||
return;
|
||||
|
||||
shutdown_called = true;
|
||||
event->set();
|
||||
if (thread.joinable())
|
||||
thread.join();
|
||||
}
|
||||
|
||||
~LeaderElection()
|
||||
@ -63,66 +67,31 @@ private:
|
||||
std::string node_name;
|
||||
|
||||
std::thread thread;
|
||||
std::atomic<bool> shutdown {false};
|
||||
std::atomic<bool> shutdown_called {false};
|
||||
zkutil::EventPtr event = std::make_shared<Poco::Event>();
|
||||
|
||||
CurrentMetrics::Increment metric_increment{CurrentMetrics::LeaderElection};
|
||||
|
||||
void createNode()
|
||||
{
|
||||
shutdown = false;
|
||||
shutdown_called = false;
|
||||
node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier);
|
||||
|
||||
std::string node_path = node->getPath();
|
||||
node_name = node_path.substr(node_path.find_last_of('/') + 1);
|
||||
|
||||
cleanOldEphemeralNodes();
|
||||
|
||||
thread = std::thread(&LeaderElection::threadFunction, this);
|
||||
}
|
||||
|
||||
void cleanOldEphemeralNodes()
|
||||
{
|
||||
if (identifier.empty())
|
||||
return;
|
||||
|
||||
/** If there are nodes with same identifier, remove them.
|
||||
* Such nodes could still be alive after failed attempt of removal,
|
||||
* if it was temporary communication failure, that was continued for more than session timeout,
|
||||
* but ZK session is still alive for unknown reason, and someone still holds that ZK session.
|
||||
* See comments in destructor of EphemeralNodeHolder.
|
||||
*/
|
||||
Strings brothers = zookeeper.getChildren(path);
|
||||
for (const auto & brother : brothers)
|
||||
{
|
||||
if (brother == node_name)
|
||||
continue;
|
||||
|
||||
std::string brother_path = path + "/" + brother;
|
||||
std::string brother_identifier = zookeeper.get(brother_path);
|
||||
|
||||
if (brother_identifier == identifier)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ObsoleteEphemeralNode);
|
||||
LOG_WARNING(&Logger::get("LeaderElection"), "Found obsolete ephemeral node for identifier "
|
||||
+ identifier + ", removing: " + brother_path);
|
||||
zookeeper.tryRemoveWithRetries(brother_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void releaseNode()
|
||||
{
|
||||
shutdown = true;
|
||||
event->set();
|
||||
if (thread.joinable())
|
||||
thread.join();
|
||||
shutdown();
|
||||
node = nullptr;
|
||||
}
|
||||
|
||||
void threadFunction()
|
||||
{
|
||||
while (!shutdown)
|
||||
while (!shutdown_called)
|
||||
{
|
||||
bool success = false;
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#include "KeeperException.h"
|
||||
#include "Lock.h"
|
||||
|
||||
|
||||
using namespace zkutil;
|
||||
|
||||
bool Lock::tryLock()
|
||||
@ -11,31 +13,16 @@ bool Lock::tryLock()
|
||||
if (tryCheck() != Status::LOCKED_BY_ME)
|
||||
locked.reset(nullptr);
|
||||
}
|
||||
|
||||
if (!locked)
|
||||
else
|
||||
{
|
||||
size_t attempt;
|
||||
std::string dummy;
|
||||
int32_t code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy);
|
||||
|
||||
/// TODO: ошибка. можно создать эфемерную ноду, но при этом не получить подтверждения даже после нескольких попыток.
|
||||
/// тогда все последующие локи будут неуспешные из-за существования ноды.
|
||||
int32_t code = zookeeper->tryCreateWithRetries(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy, &attempt);
|
||||
|
||||
if (code == ZNODEEXISTS)
|
||||
if (code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS)
|
||||
{
|
||||
if (attempt == 0)
|
||||
locked.reset(nullptr);
|
||||
else
|
||||
{
|
||||
zkutil::Stat stat;
|
||||
zookeeper->get(lock_path, &stat);
|
||||
if (stat.ephemeralOwner == zookeeper->getClientID())
|
||||
locked.reset(new ZooKeeperHandler(zookeeper));
|
||||
else
|
||||
locked.reset(nullptr);
|
||||
}
|
||||
locked.reset(nullptr);
|
||||
}
|
||||
else if (code == ZOK)
|
||||
else if (code == ZooKeeperImpl::ZooKeeper::ZOK)
|
||||
{
|
||||
locked.reset(new ZooKeeperHandler(zookeeper));
|
||||
}
|
||||
@ -52,34 +39,8 @@ void Lock::unlock()
|
||||
if (locked)
|
||||
{
|
||||
auto zookeeper = zookeeper_holder->getZooKeeper();
|
||||
try
|
||||
{
|
||||
if (tryCheck() == Status::LOCKED_BY_ME)
|
||||
{
|
||||
size_t attempt;
|
||||
int32_t code = zookeeper->tryRemoveEphemeralNodeWithRetries(lock_path, -1, &attempt);
|
||||
|
||||
if (attempt)
|
||||
{
|
||||
if (code != ZOK)
|
||||
throw zkutil::KeeperException(code);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (code == ZNONODE)
|
||||
LOG_ERROR(log, "Node " << lock_path << " has been already removed. Probably due to network error.");
|
||||
else if (code != ZOK)
|
||||
throw zkutil::KeeperException(code);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (const zkutil::KeeperException & e)
|
||||
{
|
||||
/// если сессия находится в невостанавливаемом состоянии, то эфемерные ноды нам больше не принадлежат
|
||||
/// и лок через таймаут будет отпущен
|
||||
if (!e.isUnrecoverable())
|
||||
throw;
|
||||
}
|
||||
if (tryCheck() == Status::LOCKED_BY_ME)
|
||||
zookeeper->remove(lock_path, -1);
|
||||
locked.reset(nullptr);
|
||||
}
|
||||
}
|
||||
@ -97,47 +58,17 @@ Lock::Status Lock::tryCheck() const
|
||||
else
|
||||
{
|
||||
if (stat.ephemeralOwner == zookeeper->getClientID())
|
||||
{
|
||||
lock_status = LOCKED_BY_ME;
|
||||
}
|
||||
else
|
||||
{
|
||||
lock_status = LOCKED_BY_OTHER;
|
||||
}
|
||||
}
|
||||
|
||||
if (locked && lock_status != LOCKED_BY_ME)
|
||||
LOG_WARNING(log, "Lock is lost. It is normal if session was reinitialized. Path: " << lock_path << "/" << lock_message);
|
||||
LOG_WARNING(log, "Lock is lost. It is normal if session was expired. Path: " << lock_path << "/" << lock_message);
|
||||
|
||||
return lock_status;
|
||||
}
|
||||
|
||||
std::string Lock::status2String(Status status)
|
||||
{
|
||||
if (status >= END)
|
||||
throw zkutil::KeeperException("Wrong status code: " + std::to_string(status));
|
||||
static const char * names[] = {"Unlocked", "Locked by me", "Locked by other"};
|
||||
return names[status];
|
||||
}
|
||||
|
||||
void Lock::unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks)
|
||||
{
|
||||
try
|
||||
{
|
||||
unlock();
|
||||
}
|
||||
catch (const zkutil::KeeperException & e)
|
||||
{
|
||||
if (e.isTemporaryError())
|
||||
{
|
||||
LOG_WARNING(log, "Fail to unlock lock. Move lock to vector to remove later. Path: " << getPath());
|
||||
failed_to_unlock_locks.emplace_back(std::move(*this));
|
||||
}
|
||||
else
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void Lock::unlockAssumeLockNodeRemovedManually()
|
||||
{
|
||||
locked.reset(nullptr);
|
||||
|
||||
@ -40,7 +40,7 @@ namespace zkutil
|
||||
{
|
||||
unlock();
|
||||
}
|
||||
catch (const zkutil::KeeperException & e)
|
||||
catch (...)
|
||||
{
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
@ -51,15 +51,12 @@ namespace zkutil
|
||||
UNLOCKED,
|
||||
LOCKED_BY_ME,
|
||||
LOCKED_BY_OTHER,
|
||||
END
|
||||
};
|
||||
std::string status2String(Status status);
|
||||
|
||||
/// проверяет создана ли эфемерная нода и кто ее владелец.
|
||||
Status tryCheck() const;
|
||||
|
||||
void unlock();
|
||||
void unlockOrMoveIfFailed(std::vector<zkutil::Lock> & failed_to_unlock_locks);
|
||||
void unlockAssumeLockNodeRemovedManually();
|
||||
|
||||
bool tryLock();
|
||||
|
||||
@ -3,151 +3,17 @@
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <zookeeper.h>
|
||||
#include <Common/ZooKeeper/ZooKeeperImpl.h>
|
||||
#include <Poco/Event.h>
|
||||
|
||||
|
||||
namespace zkutil
|
||||
{
|
||||
|
||||
using ACLPtr = const ACL_vector *;
|
||||
using Stat = ::Stat;
|
||||
|
||||
struct Op
|
||||
{
|
||||
public:
|
||||
Op() : data(new zoo_op_t) {}
|
||||
virtual ~Op() {}
|
||||
|
||||
virtual std::unique_ptr<Op> clone() const = 0;
|
||||
|
||||
virtual std::string getPath() const = 0;
|
||||
|
||||
virtual std::string describe() const = 0;
|
||||
|
||||
std::unique_ptr<zoo_op_t> data;
|
||||
|
||||
struct Remove;
|
||||
struct Create;
|
||||
struct SetData;
|
||||
struct Check;
|
||||
};
|
||||
|
||||
struct Op::Remove : public Op
|
||||
{
|
||||
Remove(const std::string & path_, int32_t version_) :
|
||||
path(path_), version(version_)
|
||||
{
|
||||
zoo_delete_op_init(data.get(), path.c_str(), version);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Remove(path, version));
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
|
||||
std::string describe() const override { return "command: remove, path: " + path; }
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
int32_t version;
|
||||
};
|
||||
|
||||
struct Op::Create : public Op
|
||||
{
|
||||
Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_);
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Create(path_pattern, value, acl, flags));
|
||||
}
|
||||
|
||||
std::string getPathCreated() { return created_path.data(); }
|
||||
|
||||
std::string getPath() const override { return path_pattern; }
|
||||
|
||||
std::string describe() const override
|
||||
{
|
||||
return "command: create"
|
||||
", path: " + path_pattern +
|
||||
", value: " + value;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string path_pattern;
|
||||
std::string value;
|
||||
ACLPtr acl;
|
||||
int32_t flags;
|
||||
std::vector<char> created_path;
|
||||
};
|
||||
|
||||
struct Op::SetData : public Op
|
||||
{
|
||||
SetData(const std::string & path_, const std::string & value_, int32_t version_) :
|
||||
path(path_), value(value_), version(version_)
|
||||
{
|
||||
zoo_set_op_init(data.get(), path.c_str(), value.c_str(), value.size(), version, &stat);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new SetData(path, value, version));
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
|
||||
std::string describe() const override
|
||||
{
|
||||
return
|
||||
"command: set"
|
||||
", path: " + path +
|
||||
", value: " + value +
|
||||
", version: " + std::to_string(data->set_op.version);
|
||||
}
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
std::string value;
|
||||
int32_t version;
|
||||
Stat stat;
|
||||
};
|
||||
|
||||
struct Op::Check : public Op
|
||||
{
|
||||
Check(const std::string & path_, int32_t version_) :
|
||||
path(path_), version(version_)
|
||||
{
|
||||
zoo_check_op_init(data.get(), path.c_str(), version);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Check(path, version));
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
|
||||
std::string describe() const override { return "command: check, path: " + path; }
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
int32_t version;
|
||||
};
|
||||
|
||||
struct OpResult : public zoo_op_result_t
|
||||
{
|
||||
/// Pointers in this class point to fields of class Op.
|
||||
/// Op instances have the same (or longer lifetime), therefore destructor is not required.
|
||||
};
|
||||
|
||||
using OpPtr = std::unique_ptr<Op>;
|
||||
using Ops = std::vector<OpPtr>;
|
||||
using OpResults = std::vector<OpResult>;
|
||||
using OpResultsPtr = std::shared_ptr<OpResults>;
|
||||
using Stat = ZooKeeperImpl::ZooKeeper::Stat;
|
||||
using Strings = std::vector<std::string>;
|
||||
|
||||
|
||||
namespace CreateMode
|
||||
{
|
||||
extern const int Persistent;
|
||||
@ -158,23 +24,39 @@ namespace CreateMode
|
||||
|
||||
using EventPtr = std::shared_ptr<Poco::Event>;
|
||||
|
||||
class ZooKeeper;
|
||||
|
||||
/// Callback to call when the watch fires.
|
||||
/// Because callbacks are called in the single "completion" thread internal to libzookeeper,
|
||||
/// they must execute as quickly as possible (preferably just set some notification).
|
||||
/// Parameters:
|
||||
/// zookeeper - zookeeper session to which the fired watch belongs
|
||||
/// type - event type, one of the *_EVENT constants from zookeeper.h
|
||||
/// state - session connection state, one of the *_STATE constants from zookeeper.h
|
||||
/// path - znode path to which the change happened. if event == ZOO_SESSION_EVENT it is either NULL or empty string.
|
||||
using WatchCallback = std::function<void(ZooKeeper & zookeeper, int type, int state, const char * path)>;
|
||||
using WatchCallback = ZooKeeperImpl::ZooKeeper::WatchCallback;
|
||||
|
||||
using Request = ZooKeeperImpl::ZooKeeper::Request;
|
||||
using Response = ZooKeeperImpl::ZooKeeper::Response;
|
||||
|
||||
/// Returns first op which code != ZOK or throws an exception
|
||||
/// ZooKeeper client sets correct OP codes if the transaction fails because of logical (user) errors like ZNODEEXISTS
|
||||
/// If it is failed because of network error, for example, OP codes is not set.
|
||||
/// Therefore you should make zkutil::isUserError() check before the function invocation.
|
||||
size_t getFailedOpIndex(const OpResultsPtr & op_results, int32_t transaction_return_code);
|
||||
using RequestPtr = ZooKeeperImpl::ZooKeeper::RequestPtr;
|
||||
using ResponsePtr = ZooKeeperImpl::ZooKeeper::ResponsePtr;
|
||||
|
||||
using Requests = ZooKeeperImpl::ZooKeeper::Requests;
|
||||
using Responses = ZooKeeperImpl::ZooKeeper::Responses;
|
||||
|
||||
using CreateRequest = ZooKeeperImpl::ZooKeeper::CreateRequest;
|
||||
using RemoveRequest = ZooKeeperImpl::ZooKeeper::RemoveRequest;
|
||||
using ExistsRequest = ZooKeeperImpl::ZooKeeper::ExistsRequest;
|
||||
using GetRequest = ZooKeeperImpl::ZooKeeper::GetRequest;
|
||||
using SetRequest = ZooKeeperImpl::ZooKeeper::SetRequest;
|
||||
using ListRequest = ZooKeeperImpl::ZooKeeper::ListRequest;
|
||||
using CheckRequest = ZooKeeperImpl::ZooKeeper::CheckRequest;
|
||||
|
||||
using CreateResponse = ZooKeeperImpl::ZooKeeper::CreateResponse;
|
||||
using RemoveResponse = ZooKeeperImpl::ZooKeeper::RemoveResponse;
|
||||
using ExistsResponse = ZooKeeperImpl::ZooKeeper::ExistsResponse;
|
||||
using GetResponse = ZooKeeperImpl::ZooKeeper::GetResponse;
|
||||
using SetResponse = ZooKeeperImpl::ZooKeeper::SetResponse;
|
||||
using ListResponse = ZooKeeperImpl::ZooKeeper::ListResponse;
|
||||
using CheckResponse = ZooKeeperImpl::ZooKeeper::CheckResponse;
|
||||
|
||||
RequestPtr makeCreateRequest(const std::string & path, const std::string & data, int create_mode);
|
||||
RequestPtr makeRemoveRequest(const std::string & path, int version);
|
||||
RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version);
|
||||
RequestPtr makeCheckRequest(const std::string & path, int version);
|
||||
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,17 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include "Types.h"
|
||||
#include "KeeperException.h"
|
||||
#include <Poco/Util/LayeredConfiguration.h>
|
||||
#include <unordered_set>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unistd.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
@ -29,14 +28,10 @@ namespace zkutil
|
||||
{
|
||||
|
||||
const UInt32 DEFAULT_SESSION_TIMEOUT = 30000;
|
||||
const UInt32 MEDIUM_SESSION_TIMEOUT = 120000;
|
||||
const UInt32 BIG_SESSION_TIMEOUT = 600000;
|
||||
|
||||
/// Preferred size of multi() command (in number of ops)
|
||||
constexpr size_t MULTI_BATCH_SIZE = 100;
|
||||
|
||||
struct WatchContext;
|
||||
struct MultiTransactionInfo;
|
||||
|
||||
/// ZooKeeper session. The interface is substantially different from the usual libzookeeper API.
|
||||
///
|
||||
@ -56,7 +51,7 @@ public:
|
||||
using Ptr = std::shared_ptr<ZooKeeper>;
|
||||
|
||||
ZooKeeper(const std::string & hosts, const std::string & identity = "",
|
||||
int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, bool check_root_exists = false);
|
||||
int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, const std::string & chroot = "");
|
||||
|
||||
/** Config of the form:
|
||||
<zookeeper>
|
||||
@ -77,27 +72,15 @@ public:
|
||||
*/
|
||||
ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name);
|
||||
|
||||
~ZooKeeper();
|
||||
|
||||
/// Creates a new session with the same parameters. This method can be used for reconnecting
|
||||
/// after the session has expired.
|
||||
/// This object remains unchanged, and the new session is returned.
|
||||
Ptr startNewSession() const;
|
||||
|
||||
/// Returns true, if the session has expired forever.
|
||||
/// This is possible only if the connection has been established, then lost and re-established
|
||||
/// again, but too late.
|
||||
/// In contrast, if, for instance, the server name or port is misconfigured, connection
|
||||
/// attempts will continue indefinitely, expired() will return false and all method calls
|
||||
/// will raise ConnectionLoss exception.
|
||||
/// Also returns true if is_dirty flag is set - a request to close the session ASAP.
|
||||
/// Returns true, if the session has expired.
|
||||
bool expired();
|
||||
|
||||
ACLPtr getDefaultACL();
|
||||
|
||||
void setDefaultACL(ACLPtr new_acl);
|
||||
|
||||
/// Create a znode. ACL set by setDefaultACL is used (full access to everybody by default).
|
||||
/// Create a znode.
|
||||
/// Throw an exception if something went wrong.
|
||||
std::string create(const std::string & path, const std::string & data, int32_t mode);
|
||||
|
||||
@ -108,12 +91,9 @@ public:
|
||||
/// In case of other errors throws an exception.
|
||||
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode);
|
||||
int32_t tryCreateWithRetries(const std::string & path, const std::string & data, int32_t mode,
|
||||
std::string & path_created, size_t * attempt = nullptr);
|
||||
|
||||
/// Create a Persistent node.
|
||||
/// Does nothing if the node already exists.
|
||||
/// Retries on ConnectionLoss or OperationTimeout.
|
||||
void createIfNotExists(const std::string & path, const std::string & data);
|
||||
|
||||
/// Creates all non-existent ancestors of the given path with empty contents.
|
||||
@ -123,45 +103,14 @@ public:
|
||||
/// Remove the node if the version matches. (if version == -1, remove any version).
|
||||
void remove(const std::string & path, int32_t version = -1);
|
||||
|
||||
/// Removes the node. In case of network errors tries to remove again.
|
||||
/// ZNONODE error for the second and the following tries is ignored.
|
||||
void removeWithRetries(const std::string & path, int32_t version = -1);
|
||||
|
||||
/// Doesn't throw in the following cases:
|
||||
/// * The node doesn't exist
|
||||
/// * Versions don't match
|
||||
/// * The node has children.
|
||||
int32_t tryRemove(const std::string & path, int32_t version = -1);
|
||||
/// Retries in case of network errors, returns ZNONODE if the node is already removed.
|
||||
int32_t tryRemoveWithRetries(const std::string & path, int32_t version = -1, size_t * attempt = nullptr);
|
||||
|
||||
/// The same, but sets is_dirty flag if all removal attempts were unsuccessful.
|
||||
/// This is needed because the session might still exist after all retries,
|
||||
/// even if more time than session_timeout has passed.
|
||||
/// So we do not rely on the ephemeral node being deleted and set is_dirty to
|
||||
/// try and close the session ASAP.
|
||||
/** Ridiculously Long Delay to Expire
|
||||
When disconnects do happen, the common case should be a very* quick
|
||||
reconnect to another server, but an extended network outage may
|
||||
introduce a long delay before a client can reconnect to the ZooKeep‐
|
||||
er service. Some developers wonder why the ZooKeeper client li‐
|
||||
brary doesn’t simply decide at some point (perhaps twice the session
|
||||
timeout) that enough is enough and kill the session itself.
|
||||
There are two answers to this. First, ZooKeeper leaves this kind of
|
||||
policy decision up to the developer. Developers can easily implement
|
||||
such a policy by closing the handle themselves. Second, when a Zoo‐
|
||||
Keeper ensemble goes down, time freezes. Thus, when the ensemble is
|
||||
brought back up, session timeouts are restarted. If processes using
|
||||
ZooKeeper hang in there, they may find out that the long timeout was
|
||||
due to an extended ensemble failure that has recovered and pick right
|
||||
up where they left off without any additional startup delay.
|
||||
|
||||
ZooKeeper: Distributed Process Coordination p118
|
||||
*/
|
||||
int32_t tryRemoveEphemeralNodeWithRetries(const std::string & path, int32_t version = -1, size_t * attempt = nullptr);
|
||||
|
||||
bool exists(const std::string & path, Stat * stat = nullptr, const EventPtr & watch = nullptr);
|
||||
bool existsWatch(const std::string & path, Stat * stat, const WatchCallback & watch_callback);
|
||||
bool existsWatch(const std::string & path, Stat * stat, WatchCallback watch_callback);
|
||||
|
||||
std::string get(const std::string & path, Stat * stat = nullptr, const EventPtr & watch = nullptr);
|
||||
|
||||
@ -169,7 +118,7 @@ public:
|
||||
/// * The node doesn't exist. Returns false in this case.
|
||||
bool tryGet(const std::string & path, std::string & res, Stat * stat = nullptr, const EventPtr & watch = nullptr, int * code = nullptr);
|
||||
|
||||
bool tryGetWatch(const std::string & path, std::string & res, Stat * stat, const WatchCallback & watch_callback, int * code = nullptr);
|
||||
bool tryGetWatch(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback, int * code = nullptr);
|
||||
|
||||
void set(const std::string & path, const std::string & data,
|
||||
int32_t version = -1, Stat * stat = nullptr);
|
||||
@ -195,15 +144,15 @@ public:
|
||||
|
||||
/// Performs several operations in a transaction.
|
||||
/// Throws on every error.
|
||||
OpResultsPtr multi(const Ops & ops);
|
||||
|
||||
Responses multi(const Requests & requests);
|
||||
/// Throws only if some operation has returned an "unexpected" error
|
||||
/// - an error that would cause the corresponding try- method to throw.
|
||||
int32_t tryMulti(const Ops & ops, OpResultsPtr * out_results = nullptr);
|
||||
/// Like previous one, but does not throw any ZooKeeper exceptions
|
||||
int32_t tryMultiUnsafe(const Ops & ops, MultiTransactionInfo & info);
|
||||
/// Use only with read-only operations.
|
||||
int32_t tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results = nullptr, size_t * attempt = nullptr);
|
||||
int32_t tryMulti(const Requests & requests, Responses & responses);
|
||||
/// Throws nothing, just alias of multiImpl
|
||||
int32_t tryMultiNoThrow(const Requests & requests, Responses & responses)
|
||||
{
|
||||
return multiImpl(requests, responses);
|
||||
}
|
||||
|
||||
Int64 getClientID();
|
||||
|
||||
@ -235,200 +184,56 @@ public:
|
||||
///
|
||||
/// Future should not be destroyed before the result is gotten.
|
||||
|
||||
template <typename Result, typename... TaskParams>
|
||||
class Future
|
||||
{
|
||||
friend class ZooKeeper;
|
||||
private:
|
||||
using Task = std::packaged_task<Result (TaskParams...)>;
|
||||
using TaskPtr = std::unique_ptr<Task>;
|
||||
using TaskPtrPtr = std::unique_ptr<TaskPtr>;
|
||||
std::future<ZooKeeperImpl::ZooKeeper::GetResponse> asyncGet(const std::string & path);
|
||||
|
||||
/// Everything is complicated.
|
||||
///
|
||||
/// In libzookeeper async interface a function (e.g. zoo_aget)
|
||||
/// accepts a pointer to a standalone callback function and void* pointer to the context
|
||||
/// which is then passed to the callback.
|
||||
/// The caller is responsible for ensuring that the context lives until the callback
|
||||
/// is finished and we can't simply pass ownership of the context into function object.
|
||||
/// Instead, we save the context in a Future object and return it to the caller.
|
||||
/// The context will live until the Future lives.
|
||||
/// Context data is wrapped in an unique_ptr so that its address (which is passed to
|
||||
/// libzookeeper) remains unchanged after the Future is returned from the function.
|
||||
///
|
||||
/// The second problem is that after std::promise has been fulfilled, and the user
|
||||
/// has gotten the result from std::future, the Future object can be destroyed
|
||||
/// before the std::promise::set_value() call that fulfils the promise completes in another
|
||||
/// thread.
|
||||
/// See http://stackoverflow.com/questions/10843304/race-condition-in-pthread-once
|
||||
/// To combat this we use the second unique_ptr. Inside the callback, the void* context
|
||||
/// is cast to unique_ptr and moved into the local unique_ptr to prolong the lifetime of
|
||||
/// the context data.
|
||||
std::future<ZooKeeperImpl::ZooKeeper::GetResponse> asyncTryGet(const std::string & path);
|
||||
|
||||
TaskPtrPtr task;
|
||||
std::future<Result> future;
|
||||
std::future<ZooKeeperImpl::ZooKeeper::ExistsResponse> asyncExists(const std::string & path);
|
||||
|
||||
template <typename... Args>
|
||||
Future(Args &&... args) :
|
||||
task(std::make_unique<TaskPtr>(std::make_unique<Task>(std::forward<Args>(args)...))),
|
||||
future((*task)->get_future()) {}
|
||||
std::future<ZooKeeperImpl::ZooKeeper::ListResponse> asyncGetChildren(const std::string & path);
|
||||
|
||||
public:
|
||||
Result get()
|
||||
{
|
||||
return future.get();
|
||||
}
|
||||
|
||||
Future(Future &&) = default;
|
||||
Future & operator= (Future &&) = default;
|
||||
|
||||
~Future()
|
||||
{
|
||||
/// If nobody has waited for the result, we must wait for it before the object is
|
||||
/// destroyed, because the object contents can still be used in the callback.
|
||||
if (future.valid())
|
||||
future.wait();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct ValueAndStat
|
||||
{
|
||||
std::string value;
|
||||
Stat stat;
|
||||
};
|
||||
|
||||
using GetFuture = Future<ValueAndStat, int, const char *, int, const Stat *>;
|
||||
GetFuture asyncGet(const std::string & path);
|
||||
|
||||
|
||||
struct ValueAndStatAndExists
|
||||
{
|
||||
std::string value;
|
||||
Stat stat;
|
||||
bool exists;
|
||||
};
|
||||
|
||||
using TryGetFuture = Future<ValueAndStatAndExists, int, const char *, int, const Stat *>;
|
||||
TryGetFuture asyncTryGet(const std::string & path);
|
||||
|
||||
|
||||
struct StatAndExists
|
||||
{
|
||||
Stat stat;
|
||||
bool exists;
|
||||
};
|
||||
|
||||
using ExistsFuture = Future<StatAndExists, int, const Stat *>;
|
||||
ExistsFuture asyncExists(const std::string & path);
|
||||
|
||||
|
||||
using GetChildrenFuture = Future<Strings, int, const String_vector *>;
|
||||
GetChildrenFuture asyncGetChildren(const std::string & path);
|
||||
|
||||
|
||||
using RemoveFuture = Future<void, int>;
|
||||
RemoveFuture asyncRemove(const std::string & path, int32_t version = -1);
|
||||
std::future<ZooKeeperImpl::ZooKeeper::RemoveResponse> asyncRemove(const std::string & path, int32_t version = -1);
|
||||
|
||||
/// Doesn't throw in the following cases:
|
||||
/// * The node doesn't exist
|
||||
/// * The versions do not match
|
||||
/// * The node has children
|
||||
using TryRemoveFuture = Future<int32_t, int>;
|
||||
TryRemoveFuture asyncTryRemove(const std::string & path, int32_t version = -1);
|
||||
std::future<ZooKeeperImpl::ZooKeeper::RemoveResponse> asyncTryRemove(const std::string & path, int32_t version = -1);
|
||||
|
||||
struct OpResultsAndCode
|
||||
{
|
||||
OpResultsPtr results;
|
||||
std::shared_ptr<Ops> ops_ptr;
|
||||
int code;
|
||||
};
|
||||
std::future<ZooKeeperImpl::ZooKeeper::MultiResponse> asyncMulti(const Requests & ops);
|
||||
|
||||
using MultiFuture = Future<OpResultsAndCode, int>;
|
||||
MultiFuture asyncMulti(const Ops & ops);
|
||||
/// Like the previous one but don't throw any exceptions on future.get()
|
||||
MultiFuture tryAsyncMulti(const Ops & ops);
|
||||
|
||||
std::future<ZooKeeperImpl::ZooKeeper::MultiResponse> tryAsyncMulti(const Requests & ops);
|
||||
|
||||
static std::string error2string(int32_t code);
|
||||
|
||||
/// Max size of node contents in bytes.
|
||||
/// In 3.4.5 max node size is 1Mb.
|
||||
static const size_t MAX_NODE_SIZE = 1048576;
|
||||
|
||||
/// Length of the suffix that ZooKeeper adds to sequential nodes.
|
||||
/// In fact it is smaller, but round it up for convenience.
|
||||
static const size_t SEQUENTIAL_SUFFIX_SIZE = 64;
|
||||
|
||||
|
||||
zhandle_t * getHandle() { return impl; }
|
||||
|
||||
private:
|
||||
friend struct WatchContext;
|
||||
friend class EphemeralNodeHolder;
|
||||
|
||||
void init(const std::string & hosts, const std::string & identity,
|
||||
int32_t session_timeout_ms, bool check_root_exists);
|
||||
void init(const std::string & hosts_, const std::string & identity_, int32_t session_timeout_ms_, const std::string & chroot_);
|
||||
|
||||
void removeChildrenRecursive(const std::string & path);
|
||||
void tryRemoveChildrenRecursive(const std::string & path);
|
||||
|
||||
static WatchCallback callbackForEvent(const EventPtr & event);
|
||||
WatchContext * createContext(WatchCallback && callback);
|
||||
static void destroyContext(WatchContext * context);
|
||||
static void processCallback(zhandle_t * zh, int type, int state, const char * path, void * watcher_ctx);
|
||||
|
||||
template <typename T>
|
||||
int32_t retry(T && operation, size_t * attempt = nullptr)
|
||||
{
|
||||
int32_t code = operation();
|
||||
if (attempt)
|
||||
*attempt = 0;
|
||||
for (size_t i = 0; (i < retry_num) && (code == ZOPERATIONTIMEOUT || code == ZCONNECTIONLOSS); ++i)
|
||||
{
|
||||
if (attempt)
|
||||
*attempt = i;
|
||||
|
||||
/// If the connection has been lost, wait timeout/3 hoping for connection re-establishment.
|
||||
static const int MAX_SLEEP_TIME = 10;
|
||||
if (code == ZCONNECTIONLOSS)
|
||||
usleep(std::min(session_timeout_ms * 1000 / 3, MAX_SLEEP_TIME * 1000 * 1000));
|
||||
|
||||
LOG_WARNING(log, "Error on attempt " << i << ": " << error2string(code) << ". Retry");
|
||||
code = operation();
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
/// The following methods don't throw exceptions but return error codes.
|
||||
int32_t createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created);
|
||||
int32_t removeImpl(const std::string & path, int32_t version = -1);
|
||||
int32_t removeImpl(const std::string & path, int32_t version);
|
||||
int32_t getImpl(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback);
|
||||
int32_t setImpl(const std::string & path, const std::string & data, int32_t version = -1, Stat * stat = nullptr);
|
||||
int32_t setImpl(const std::string & path, const std::string & data, int32_t version, Stat * stat);
|
||||
int32_t getChildrenImpl(const std::string & path, Strings & res, Stat * stat, WatchCallback watch_callback);
|
||||
int32_t multiImpl(const Ops & ops, OpResultsPtr * out_results = nullptr);
|
||||
int32_t multiImpl(const Requests & requests, Responses & responses);
|
||||
int32_t existsImpl(const std::string & path, Stat * stat_, WatchCallback watch_callback);
|
||||
|
||||
MultiFuture asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception);
|
||||
std::unique_ptr<ZooKeeperImpl::ZooKeeper> impl;
|
||||
|
||||
std::string hosts;
|
||||
std::string identity;
|
||||
int32_t session_timeout_ms;
|
||||
std::string chroot;
|
||||
|
||||
std::mutex mutex;
|
||||
ACLPtr default_acl;
|
||||
zhandle_t * impl;
|
||||
|
||||
std::unordered_set<WatchContext *> watch_context_store;
|
||||
|
||||
/// Retries number in case of OperationTimeout or ConnectionLoss errors.
|
||||
static constexpr size_t retry_num = 3;
|
||||
Logger * log = nullptr;
|
||||
|
||||
/// If true, there were unsuccessfull attempts to remove ephemeral nodes.
|
||||
/// It is better to close the session to remove ephemeral nodes with certainty
|
||||
/// instead of continuing to use re-established session.
|
||||
bool is_dirty = false;
|
||||
};
|
||||
|
||||
|
||||
@ -472,13 +277,9 @@ public:
|
||||
{
|
||||
try
|
||||
{
|
||||
/// Important: if the ZooKeeper is temporarily unavailable, repeated attempts to
|
||||
/// delete the node are made.
|
||||
/// Otherwise it is possible that EphemeralNodeHolder is destroyed,
|
||||
/// but the session has recovered and the node in ZooKeeper remains for the long time.
|
||||
zookeeper.tryRemoveEphemeralNodeWithRetries(path);
|
||||
zookeeper.tryRemove(path);
|
||||
}
|
||||
catch (const KeeperException & e)
|
||||
catch (...)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::CannotRemoveEphemeralNode);
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
@ -493,41 +294,4 @@ private:
|
||||
|
||||
using EphemeralNodeHolderPtr = EphemeralNodeHolder::Ptr;
|
||||
|
||||
|
||||
/// Simple structure to handle transaction execution results
|
||||
struct MultiTransactionInfo
|
||||
{
|
||||
MultiTransactionInfo() = default;
|
||||
|
||||
Ops ops;
|
||||
int32_t code = ZOK;
|
||||
OpResultsPtr op_results;
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return ops.empty();
|
||||
}
|
||||
|
||||
bool hasFailedOp() const
|
||||
{
|
||||
return zkutil::isUserError(code);
|
||||
}
|
||||
|
||||
const Op & getFailedOp() const
|
||||
{
|
||||
return *ops.at(getFailedOpIndex(op_results, code));
|
||||
}
|
||||
|
||||
KeeperException getException() const
|
||||
{
|
||||
if (hasFailedOp())
|
||||
{
|
||||
size_t i = getFailedOpIndex(op_results, code);
|
||||
return KeeperException("Transaction failed at op #" + std::to_string(i) + ": " + ops.at(i)->describe(), code);
|
||||
}
|
||||
else
|
||||
return KeeperException(code);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
1541
dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp
Normal file
1541
dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp
Normal file
File diff suppressed because it is too large
Load Diff
635
dbms/src/Common/ZooKeeper/ZooKeeperImpl.h
Normal file
635
dbms/src/Common/ZooKeeper/ZooKeeperImpl.h
Normal file
@ -0,0 +1,635 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/ReadBufferFromPocoSocket.h>
|
||||
#include <IO/WriteBufferFromPocoSocket.h>
|
||||
|
||||
#include <Poco/Net/StreamSocket.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <chrono>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <functional>
|
||||
|
||||
|
||||
/** ZooKeeper C++ library, a replacement for libzookeeper.
|
||||
*
|
||||
* Motivation.
|
||||
*
|
||||
* libzookeeper has many bugs:
|
||||
* - segfaults: for example, if zookeeper connection was interrupted while reading result of multi response;
|
||||
* - memory corruption: for example, as a result of double free inside libzookeeper;
|
||||
* - no timeouts for synchronous operations: they may stuck forever under simple Jepsen-like tests;
|
||||
* - logical errors: for example, chroot prefix is not removed from the results of multi responses.
|
||||
* - data races;
|
||||
*
|
||||
* The code of libzookeeper is over complicated:
|
||||
* - memory ownership is unclear and bugs are very difficult to track and fix.
|
||||
* - extremely creepy code for implementation of "chroot" feature.
|
||||
*
|
||||
* As of 2018, there are no active maintainers of libzookeeper:
|
||||
* - bugs in JIRA are fixed only occasionaly with ad-hoc patches by library users.
|
||||
*
|
||||
* libzookeeper is a classical example of bad code written in C.
|
||||
*
|
||||
* In Go, Python and Rust programming languages,
|
||||
* there are separate libraries for ZooKeeper, not based on libzookeeper.
|
||||
* Motivation is almost the same. Example:
|
||||
* https://github.com/python-zk/kazoo/blob/master/docs/implementation.rst
|
||||
*
|
||||
* About "session restore" feature.
|
||||
*
|
||||
* libzookeeper has the feature of session restore. Client receives session id and session token from the server,
|
||||
* and when connection is lost, it can quickly reconnect to any server with the same session id and token,
|
||||
* to continue with existing session.
|
||||
* libzookeeper performs this reconnection automatically.
|
||||
*
|
||||
* This feature is proven to be harmful.
|
||||
* For example, it makes very difficult to correctly remove ephemeral nodes.
|
||||
* This may lead to weird bugs in application code.
|
||||
* For example, our developers have found that type of bugs in Curator Java library.
|
||||
*
|
||||
* On the other side, session restore feature has no advantages,
|
||||
* because every application should be able to establish new session and reinitialize internal state,
|
||||
* when the session is lost and cannot be restored.
|
||||
*
|
||||
* This library never restores the session. In case of any error, the session is considered as expired
|
||||
* and you should create a new instance of ZooKeeper object and reinitialize the application state.
|
||||
*
|
||||
* This library is not intended to be CPU efficient. Hundreds of thousands operations per second is usually enough.
|
||||
*/
|
||||
|
||||
|
||||
namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric ZooKeeperSession;
|
||||
}
|
||||
|
||||
|
||||
namespace ZooKeeperImpl
|
||||
{
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
class Exception : public DB::Exception
|
||||
{
|
||||
private:
|
||||
/// Delegate constructor, used to minimize repetition; last parameter used for overload resolution.
|
||||
Exception(const std::string & msg, const int32_t code, int);
|
||||
|
||||
public:
|
||||
explicit Exception(const int32_t code);
|
||||
Exception(const std::string & msg, const int32_t code);
|
||||
Exception(const int32_t code, const std::string & path);
|
||||
Exception(const Exception & exc);
|
||||
|
||||
const char * name() const throw() override { return "ZooKeeperImpl::Exception"; }
|
||||
const char * className() const throw() override { return "ZooKeeperImpl::Exception"; }
|
||||
Exception * clone() const override { return new Exception(*this); }
|
||||
|
||||
const int32_t code;
|
||||
};
|
||||
|
||||
|
||||
/** Usage scenario:
|
||||
* - create an object and issue commands;
|
||||
* - you provide callbacks for your commands; callbacks are invoked in internal thread and must be cheap:
|
||||
* for example, just signal a condvar / fulfull a promise.
|
||||
* - you also may provide callbacks for watches; they are also invoked in internal thread and must be cheap.
|
||||
* - whenever you receive exception with ZSESSIONEXPIRED code or method isExpired returns true,
|
||||
* the ZooKeeper instance is no longer usable - you may only destroy it and probably create another.
|
||||
* - whenever session is expired or ZooKeeper instance is destroying, all callbacks are notified with special event.
|
||||
* - data for callbacks must be alive when ZooKeeper instance is alive.
|
||||
*/
|
||||
class ZooKeeper
|
||||
{
|
||||
public:
|
||||
using Addresses = std::vector<Poco::Net::SocketAddress>;
|
||||
|
||||
struct ACL
|
||||
{
|
||||
static constexpr int32_t Read = 1;
|
||||
static constexpr int32_t Write = 2;
|
||||
static constexpr int32_t Create = 4;
|
||||
static constexpr int32_t Delete = 8;
|
||||
static constexpr int32_t Admin = 16;
|
||||
static constexpr int32_t All = 0x1F;
|
||||
|
||||
int32_t permissions;
|
||||
String scheme;
|
||||
String id;
|
||||
|
||||
void write(WriteBuffer & out) const;
|
||||
};
|
||||
using ACLs = std::vector<ACL>;
|
||||
|
||||
struct Stat
|
||||
{
|
||||
int64_t czxid;
|
||||
int64_t mzxid;
|
||||
int64_t ctime;
|
||||
int64_t mtime;
|
||||
int32_t version;
|
||||
int32_t cversion;
|
||||
int32_t aversion;
|
||||
int64_t ephemeralOwner;
|
||||
int32_t dataLength;
|
||||
int32_t numChildren;
|
||||
int64_t pzxid;
|
||||
|
||||
void read(ReadBuffer & in);
|
||||
};
|
||||
|
||||
using XID = int32_t;
|
||||
using OpNum = int32_t;
|
||||
|
||||
|
||||
struct Response
|
||||
{
|
||||
int32_t error = 0;
|
||||
virtual ~Response() {}
|
||||
virtual void readImpl(ReadBuffer &) = 0;
|
||||
|
||||
virtual void removeRootPath(const String & /* root_path */) {}
|
||||
};
|
||||
|
||||
using ResponsePtr = std::shared_ptr<Response>;
|
||||
using Responses = std::vector<ResponsePtr>;
|
||||
using ResponseCallback = std::function<void(const Response &)>;
|
||||
|
||||
struct Request
|
||||
{
|
||||
XID xid = 0;
|
||||
bool has_watch = false;
|
||||
|
||||
virtual ~Request() {};
|
||||
virtual OpNum getOpNum() const = 0;
|
||||
|
||||
/// Writes length, xid, op_num, then the rest.
|
||||
void write(WriteBuffer & out) const;
|
||||
virtual void writeImpl(WriteBuffer &) const = 0;
|
||||
|
||||
virtual ResponsePtr makeResponse() const = 0;
|
||||
|
||||
virtual void addRootPath(const String & /* root_path */) {};
|
||||
virtual String getPath() const = 0;
|
||||
};
|
||||
|
||||
using RequestPtr = std::shared_ptr<Request>;
|
||||
using Requests = std::vector<RequestPtr>;
|
||||
|
||||
struct HeartbeatRequest final : Request
|
||||
{
|
||||
OpNum getOpNum() const override { return 11; }
|
||||
void writeImpl(WriteBuffer &) const override {}
|
||||
ResponsePtr makeResponse() const override;
|
||||
String getPath() const override { return {}; }
|
||||
};
|
||||
|
||||
struct HeartbeatResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override {}
|
||||
};
|
||||
|
||||
struct WatchResponse final : Response
|
||||
{
|
||||
int32_t type = 0;
|
||||
int32_t state = 0;
|
||||
String path;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
void removeRootPath(const String & root_path) override;
|
||||
};
|
||||
|
||||
using WatchCallback = std::function<void(const WatchResponse &)>;
|
||||
|
||||
struct AuthRequest final : Request
|
||||
{
|
||||
int32_t type = 0; /// ignored by the server
|
||||
String scheme;
|
||||
String data;
|
||||
|
||||
OpNum getOpNum() const override { return 100; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
String getPath() const override { return {}; }
|
||||
};
|
||||
|
||||
struct AuthResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override {};
|
||||
};
|
||||
|
||||
struct CloseRequest final : Request
|
||||
{
|
||||
OpNum getOpNum() const override { return -11; }
|
||||
void writeImpl(WriteBuffer &) const override {}
|
||||
ResponsePtr makeResponse() const override;
|
||||
String getPath() const override { return {}; }
|
||||
};
|
||||
|
||||
struct CloseResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
struct CreateRequest final : Request
|
||||
{
|
||||
String path;
|
||||
String data;
|
||||
bool is_ephemeral = false;
|
||||
bool is_sequential = false;
|
||||
ACLs acls;
|
||||
|
||||
OpNum getOpNum() const override { return 1; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct CreateResponse final : Response
|
||||
{
|
||||
String path_created;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
void removeRootPath(const String & root_path) override;
|
||||
};
|
||||
|
||||
struct RemoveRequest final : Request
|
||||
{
|
||||
String path;
|
||||
int32_t version = -1;
|
||||
|
||||
OpNum getOpNum() const override { return 2; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct RemoveResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override {}
|
||||
};
|
||||
|
||||
struct ExistsRequest final : Request
|
||||
{
|
||||
String path;
|
||||
|
||||
OpNum getOpNum() const override { return 3; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct ExistsResponse final : Response
|
||||
{
|
||||
Stat stat;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
struct GetRequest final : Request
|
||||
{
|
||||
String path;
|
||||
|
||||
OpNum getOpNum() const override { return 4; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct GetResponse final : Response
|
||||
{
|
||||
String data;
|
||||
Stat stat;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
struct SetRequest final : Request
|
||||
{
|
||||
String path;
|
||||
String data;
|
||||
int32_t version = -1;
|
||||
|
||||
OpNum getOpNum() const override { return 5; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct SetResponse final : Response
|
||||
{
|
||||
Stat stat;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
struct ListRequest final : Request
|
||||
{
|
||||
String path;
|
||||
|
||||
OpNum getOpNum() const override { return 12; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct ListResponse final : Response
|
||||
{
|
||||
std::vector<String> names;
|
||||
Stat stat;
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
struct CheckRequest final : Request
|
||||
{
|
||||
String path;
|
||||
int32_t version = -1;
|
||||
|
||||
OpNum getOpNum() const override { return 13; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return path; }
|
||||
};
|
||||
|
||||
struct CheckResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override {};
|
||||
};
|
||||
|
||||
struct MultiRequest final : Request
|
||||
{
|
||||
Requests requests;
|
||||
|
||||
OpNum getOpNum() const override { return 14; }
|
||||
void writeImpl(WriteBuffer &) const override;
|
||||
ResponsePtr makeResponse() const override;
|
||||
void addRootPath(const String & root_path) override;
|
||||
String getPath() const override { return {}; }
|
||||
};
|
||||
|
||||
struct MultiResponse final : Response
|
||||
{
|
||||
Responses responses;
|
||||
|
||||
MultiResponse(const Requests & requests);
|
||||
|
||||
void readImpl(ReadBuffer &) override;
|
||||
void removeRootPath(const String & root_path) override;
|
||||
};
|
||||
|
||||
/// This response may be received only as an element of responses in MultiResponse.
|
||||
struct ErrorResponse final : Response
|
||||
{
|
||||
void readImpl(ReadBuffer &) override;
|
||||
};
|
||||
|
||||
|
||||
/** Connection to addresses is performed in order. If you want, shuffle them manually.
|
||||
* Operation timeout couldn't be greater than session timeout.
|
||||
* Operation timeout applies independently for network read, network write, waiting for events and synchronization.
|
||||
*/
|
||||
ZooKeeper(
|
||||
const Addresses & addresses,
|
||||
const String & root_path,
|
||||
const String & auth_scheme,
|
||||
const String & auth_data,
|
||||
Poco::Timespan session_timeout,
|
||||
Poco::Timespan connection_timeout,
|
||||
Poco::Timespan operation_timeout);
|
||||
|
||||
~ZooKeeper();
|
||||
|
||||
|
||||
/// If expired, you can only destroy the object. All other methods will throw exception.
|
||||
bool isExpired() const { return expired; }
|
||||
|
||||
/// Useful to check owner of ephemeral node.
|
||||
int64_t getSessionID() const { return session_id; }
|
||||
|
||||
|
||||
using CreateCallback = std::function<void(const CreateResponse &)>;
|
||||
using RemoveCallback = std::function<void(const RemoveResponse &)>;
|
||||
using ExistsCallback = std::function<void(const ExistsResponse &)>;
|
||||
using GetCallback = std::function<void(const GetResponse &)>;
|
||||
using SetCallback = std::function<void(const SetResponse &)>;
|
||||
using ListCallback = std::function<void(const ListResponse &)>;
|
||||
using CheckCallback = std::function<void(const CheckResponse &)>;
|
||||
using MultiCallback = std::function<void(const MultiResponse &)>;
|
||||
|
||||
/// If the method will throw an exception, callbacks won't be called.
|
||||
///
|
||||
/// After the method is executed successfully, you must wait for callbacks
|
||||
/// (don't destroy callback data before it will be called).
|
||||
///
|
||||
/// All callbacks are executed sequentially (the execution of callbacks is serialized).
|
||||
///
|
||||
/// If an exception is thrown inside the callback, the session will expire,
|
||||
/// and all other callbacks will be called with "Session expired" error.
|
||||
|
||||
void create(
|
||||
const String & path,
|
||||
const String & data,
|
||||
bool is_ephemeral,
|
||||
bool is_sequential,
|
||||
const ACLs & acls,
|
||||
CreateCallback callback);
|
||||
|
||||
void remove(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
RemoveCallback callback);
|
||||
|
||||
void exists(
|
||||
const String & path,
|
||||
ExistsCallback callback,
|
||||
WatchCallback watch);
|
||||
|
||||
void get(
|
||||
const String & path,
|
||||
GetCallback callback,
|
||||
WatchCallback watch);
|
||||
|
||||
void set(
|
||||
const String & path,
|
||||
const String & data,
|
||||
int32_t version,
|
||||
SetCallback callback);
|
||||
|
||||
void list(
|
||||
const String & path,
|
||||
ListCallback callback,
|
||||
WatchCallback watch);
|
||||
|
||||
void check(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
CheckCallback callback);
|
||||
|
||||
void multi(
|
||||
const Requests & requests,
|
||||
MultiCallback callback);
|
||||
|
||||
|
||||
enum Error
|
||||
{
|
||||
ZOK = 0,
|
||||
|
||||
/** System and server-side errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value, but lesser than ZAPIERROR, are system errors.
|
||||
*/
|
||||
ZSYSTEMERROR = -1,
|
||||
|
||||
ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found
|
||||
ZDATAINCONSISTENCY = -3, /// A data inconsistency was found
|
||||
ZCONNECTIONLOSS = -4, /// Connection to the server has been lost
|
||||
ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data
|
||||
ZUNIMPLEMENTED = -6, /// Operation is unimplemented
|
||||
ZOPERATIONTIMEOUT = -7, /// Operation timeout
|
||||
ZBADARGUMENTS = -8, /// Invalid arguments
|
||||
ZINVALIDSTATE = -9, /// Invliad zhandle state
|
||||
|
||||
/** API errors.
|
||||
* This is never thrown by the server, it shouldn't be used other than
|
||||
* to indicate a range. Specifically error codes greater than this
|
||||
* value are API errors.
|
||||
*/
|
||||
ZAPIERROR = -100,
|
||||
|
||||
ZNONODE = -101, /// Node does not exist
|
||||
ZNOAUTH = -102, /// Not authenticated
|
||||
ZBADVERSION = -103, /// Version conflict
|
||||
ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children
|
||||
ZNODEEXISTS = -110, /// The node already exists
|
||||
ZNOTEMPTY = -111, /// The node has children
|
||||
ZSESSIONEXPIRED = -112, /// The session has been expired by the server
|
||||
ZINVALIDCALLBACK = -113, /// Invalid callback specified
|
||||
ZINVALIDACL = -114, /// Invalid ACL specified
|
||||
ZAUTHFAILED = -115, /// Client authentication failed
|
||||
ZCLOSING = -116, /// ZooKeeper is closing
|
||||
ZNOTHING = -117, /// (not error) no server responses to process
|
||||
ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored
|
||||
};
|
||||
|
||||
static const char * errorMessage(int32_t code);
|
||||
|
||||
/// For watches.
|
||||
enum State
|
||||
{
|
||||
EXPIRED_SESSION = -112,
|
||||
AUTH_FAILED = -113,
|
||||
CONNECTING = 1,
|
||||
ASSOCIATING = 2,
|
||||
CONNECTED = 3,
|
||||
NOTCONNECTED = 999
|
||||
};
|
||||
|
||||
enum Event
|
||||
{
|
||||
CREATED = 1,
|
||||
DELETED = 2,
|
||||
CHANGED = 3,
|
||||
CHILD = 4,
|
||||
SESSION = -1,
|
||||
NOTWATCHING = -2
|
||||
};
|
||||
|
||||
private:
|
||||
String root_path;
|
||||
ACLs default_acls;
|
||||
|
||||
Poco::Timespan session_timeout;
|
||||
Poco::Timespan operation_timeout;
|
||||
|
||||
Poco::Net::StreamSocket socket;
|
||||
std::optional<ReadBufferFromPocoSocket> in;
|
||||
std::optional<WriteBufferFromPocoSocket> out;
|
||||
|
||||
int64_t session_id = 0;
|
||||
|
||||
std::atomic<XID> xid {1};
|
||||
std::atomic<bool> expired {false};
|
||||
std::mutex finalize_mutex;
|
||||
|
||||
using clock = std::chrono::steady_clock;
|
||||
|
||||
struct RequestInfo
|
||||
{
|
||||
RequestPtr request;
|
||||
ResponseCallback callback;
|
||||
WatchCallback watch;
|
||||
clock::time_point time;
|
||||
};
|
||||
|
||||
using RequestsQueue = ConcurrentBoundedQueue<RequestInfo>;
|
||||
|
||||
RequestsQueue requests_queue{1};
|
||||
void pushRequest(RequestInfo && request);
|
||||
|
||||
using Operations = std::map<XID, RequestInfo>;
|
||||
|
||||
Operations operations;
|
||||
std::mutex operations_mutex;
|
||||
|
||||
using WatchCallbacks = std::vector<WatchCallback>;
|
||||
using Watches = std::map<String /* path */, WatchCallbacks>;
|
||||
|
||||
Watches watches;
|
||||
std::mutex watches_mutex;
|
||||
|
||||
std::thread send_thread;
|
||||
std::thread receive_thread;
|
||||
|
||||
void connect(
|
||||
const Addresses & addresses,
|
||||
Poco::Timespan connection_timeout);
|
||||
|
||||
void sendHandshake();
|
||||
void receiveHandshake();
|
||||
|
||||
void sendAuth(const String & scheme, const String & data);
|
||||
|
||||
void receiveEvent();
|
||||
|
||||
void sendThread();
|
||||
void receiveThread();
|
||||
|
||||
void close();
|
||||
|
||||
/// Call all remaining callbacks and watches, passing errors to them.
|
||||
void finalize(bool error_send, bool error_receive);
|
||||
|
||||
template <typename T>
|
||||
void write(const T &);
|
||||
|
||||
template <typename T>
|
||||
void read(T &);
|
||||
|
||||
CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession};
|
||||
};
|
||||
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user