mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-17 11:52:27 +00:00
merge
This commit is contained in:
Ivan Blinkov
commit
284a9eacc4
@ -13,7 +13,7 @@ matrix:
|
||||
# apt:
|
||||
# sources:
|
||||
# - ubuntu-toolchain-r-test
|
||||
# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
|
||||
# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ]
|
||||
#
|
||||
# env:
|
||||
# - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7"
|
||||
@ -36,7 +36,7 @@ matrix:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
- llvm-toolchain-trusty-5.0
|
||||
packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
|
||||
packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
|
||||
|
||||
env:
|
||||
- MATRIX_EVAL="export CC=clang-5.0 && export CXX=clang++-5.0"
|
||||
|
||||
29
CHANGELOG.md
29
CHANGELOG.md
@ -1,4 +1,31 @@
|
||||
# ClickHouse 1.1.54356 Release Candidate, 2018-03-06
|
||||
# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
|
||||
|
||||
## New features:
|
||||
|
||||
* Added the `system.macros` table and auto updating of macros when the config file is changed.
|
||||
* Added the `SYSTEM RELOAD CONFIG` query.
|
||||
* Added the `maxIntersections(left_col, right_col)` aggregate function, which returns the maximum number of simultaneously intersecting intervals `[left; right]`. The `maxIntersectionsPosition(left, right)` function returns the beginning of the "maximum" interval. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
|
||||
|
||||
## Improvements:
|
||||
|
||||
* When inserting data in a `Replicated` table, fewer requests are made to `ZooKeeper` (and most of the user-level errors have disappeared from the `ZooKeeper` log).
|
||||
* Added the ability to create aliases for sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
|
||||
|
||||
## Bug fixes:
|
||||
|
||||
* Fixed the `Illegal PREWHERE` error when reading from `Merge` tables over `Distributed` tables.
|
||||
* Added fixes that allow you to run `clickhouse-server` in IPv4-only Docker containers.
|
||||
* Fixed a race condition when reading from system `system.parts_columns` tables.
|
||||
* Removed double buffering during a synchronous insert to a `Distributed` table, which could have caused the connection to timeout.
|
||||
* Fixed a bug that caused excessively long waits for an unavailable replica before beginning a `SELECT` query.
|
||||
* Fixed incorrect dates in the `system.parts` table.
|
||||
* Fixed a bug that made it impossible to insert data in a `Replicated` table if `chroot` was non-empty in the configuration of the `ZooKeeper` cluster.
|
||||
* Fixed the vertical merging algorithm for an empty `ORDER BY` table.
|
||||
* Restored the ability to use dictionaries in queries to remote tables, even if these dictionaries are not present on the requestor server. This functionality was lost in release 1.1.54362.
|
||||
* Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side argument of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358.
|
||||
* Removed extraneous error-level logging of `Not found column ... in block`.
|
||||
|
||||
# ClickHouse release 1.1.54356, 2018-03-06
|
||||
|
||||
## New features:
|
||||
|
||||
|
||||
@ -1,4 +1,32 @@
|
||||
# ClickHouse 1.1.54362 Release Candidate, 2018-03-11
|
||||
# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
|
||||
|
||||
## Новые возможности:
|
||||
|
||||
* Добавлена системная таблица `system.macros` и автоматическое обновление макросов при изменении конфигурационного файла.
|
||||
* Добавлен запрос `SYSTEM RELOAD CONFIG`.
|
||||
* Добавлена агрегатная функция `maxIntersections(left_col, right_col)`, возвращающая максимальное количество одновременно пересекающихся интервалов `[left; right]`. Функция `maxIntersectionsPosition(left, right)` возвращает начало такого "максимального" интервала. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
|
||||
|
||||
## Улучшения:
|
||||
|
||||
* При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок).
|
||||
* Добавлена возможность создавать алиасы для множеств. Пример: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
|
||||
|
||||
## Исправление ошибок:
|
||||
|
||||
* Исправлена ошибка `Illegal PREWHERE` при чтении из Merge-таблицы над `Distributed`-таблицами.
|
||||
* Добавлены исправления, позволяющие запускать clickhouse-server в IPv4-only docker-контейнерах.
|
||||
* Исправлен race condition при чтении из системной таблицы `system.parts_columns`
|
||||
* Убрана двойная буферизация при синхронной вставке в `Distributed`-таблицу, которая могла приводить к timeout-ам соединений.
|
||||
* Исправлена ошибка, вызывающая чрезмерно долгое ожидание недоступной реплики перед началом выполнения `SELECT`.
|
||||
* Исправлено некорректное отображение дат в таблице `system.parts`.
|
||||
* Исправлена ошибка, приводящая к невозможности вставить данные в `Replicated`-таблицу, если в конфигурации кластера `ZooKeeper` задан непустой `chroot`.
|
||||
* Исправлен алгоритм вертикального мержа при пустом ключе `ORDER BY` таблицы.
|
||||
* Возвращена возможность использовать словари в запросах к удаленным таблицам, даже если этих словарей нет на сервере-инициаторе. Данная функциональность была потеряна в версии 1.1.54362.
|
||||
* Восстановлено поведение, при котором в запросах типа `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` в правой части `IN` должна использоваться удаленная таблица `default.table`, а не локальная. данное поведение было нарушено в версии 1.1.54358.
|
||||
* Устранено ненужное Error-level логирование `Not found column ... in block`.
|
||||
|
||||
|
||||
# Релиз ClickHouse 1.1.54362, 2018-03-11
|
||||
|
||||
## Новые возможности:
|
||||
|
||||
|
||||
@ -49,7 +49,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
|
||||
set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
|
||||
endif ()
|
||||
|
||||
if (OPENSSL_FOUND)
|
||||
# TODO! fix internal ssl
|
||||
if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
|
||||
set (Poco_NetSSL_FOUND 1)
|
||||
set (Poco_NetSSL_LIBRARY PocoNetSSL)
|
||||
set (Poco_Crypto_LIBRARY PocoCrypto)
|
||||
|
||||
22
contrib/CMakeLists.txt
vendored
22
contrib/CMakeLists.txt
vendored
@ -26,7 +26,7 @@ if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
add_subdirectory (zookeeper/src/c)
|
||||
add_subdirectory (zookeeper-cmake)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_CITYHASH_LIBRARY)
|
||||
@ -100,13 +100,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
|
||||
mark_as_advanced (ZLIB_INCLUDE_DIR)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
|
||||
set (WITH_BUNDLED_SSL 1)
|
||||
if (MAKE_STATIC_LIBRARIES)
|
||||
add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
|
||||
set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
|
||||
else ()
|
||||
set (WITH_SSL 0 CACHE INTERNAL "")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
add_subdirectory (librdkafka)
|
||||
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
|
||||
@ -127,16 +131,18 @@ endif ()
|
||||
|
||||
|
||||
if (USE_INTERNAL_POCO_LIBRARY)
|
||||
set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
|
||||
set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
|
||||
set (_save ${ENABLE_TESTS})
|
||||
set (ENABLE_TESTS 0)
|
||||
set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
|
||||
if (USE_INTERNAL_SSL_LIBRARY)
|
||||
set (DISABLE_INTERNAL_OPENSSL 1)
|
||||
set (ENABLE_NETSSL 0) # TODO!
|
||||
set (ENABLE_CRYPTO 0) # TODO!
|
||||
set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
|
||||
set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
|
||||
set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
|
||||
endif ()
|
||||
if (MSVC)
|
||||
set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
|
||||
endif ()
|
||||
add_subdirectory (poco)
|
||||
unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)
|
||||
|
||||
2
contrib/boost
vendored
2
contrib/boost
vendored
@ -1 +1 @@
|
||||
Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e
|
||||
Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded
|
||||
@ -5,7 +5,11 @@
|
||||
/* #undef AC_APPLE_UNIVERSAL_BUILD */
|
||||
|
||||
/* Define to 1 if the compiler supports __builtin_expect. */
|
||||
#if _MSC_VER
|
||||
#define HAVE_BUILTIN_EXPECT 0
|
||||
#else
|
||||
#define HAVE_BUILTIN_EXPECT 1
|
||||
#endif
|
||||
|
||||
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||
#define HAVE_DLFCN_H 1
|
||||
|
||||
@ -2,4 +2,8 @@ add_library(farmhash
|
||||
farmhash.cc
|
||||
farmhash.h)
|
||||
|
||||
if (MSVC)
|
||||
target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1)
|
||||
endif ()
|
||||
|
||||
target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
@ -13,7 +13,11 @@
|
||||
/* #undef ENABLE_SIZED_DELETE */
|
||||
|
||||
/* Define to 1 if compiler supports __builtin_expect */
|
||||
#if _MSC_VER
|
||||
#define HAVE_BUILTIN_EXPECT 0
|
||||
#else
|
||||
#define HAVE_BUILTIN_EXPECT 1
|
||||
#endif
|
||||
|
||||
/* Define to 1 if compiler supports __builtin_stack_pointer */
|
||||
/* #undef HAVE_BUILTIN_STACK_POINTER */
|
||||
|
||||
2
contrib/poco
vendored
2
contrib/poco
vendored
@ -1 +1 @@
|
||||
Subproject commit 8238852d7ab2a4abdf87adff233b3b83686f4fe4
|
||||
Subproject commit a107b0c9cee109fe0abfbf509df3c78a1e0c05fa
|
||||
200
contrib/zookeeper-cmake/CMakeLists.txt
Normal file
200
contrib/zookeeper-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,200 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
cmake_minimum_required(VERSION 3.0)
|
||||
|
||||
# Modified version of CMakeLists.txt for ClickHouse. Doesn't link the library to libm.
|
||||
# Otherwise we have extra dependency when compiling with the most fresh libc version.
|
||||
# How to check:
|
||||
# readelf -s ./clickhouse | grep -F 2.23
|
||||
|
||||
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/zookeeper/src/c)
|
||||
|
||||
project(zookeeper VERSION 3.5.3)
|
||||
set(email user@zookeeper.apache.org)
|
||||
set(description "zookeeper C client")
|
||||
|
||||
# general options
|
||||
if(UNIX)
|
||||
add_compile_options(-Wall -fPIC)
|
||||
elseif(WIN32)
|
||||
add_compile_options(/W3)
|
||||
endif()
|
||||
add_definitions(-DUSE_STATIC_LIB)
|
||||
|
||||
# TODO: Enable /WX and /W4 on Windows. Currently there are ~1000 warnings.
|
||||
# TODO: Add Solaris support.
|
||||
# TODO: Add a shared library option.
|
||||
# TODO: Specify symbols to export.
|
||||
# TODO: Generate doxygen documentation.
|
||||
|
||||
# Sync API option
|
||||
option(WANT_SYNCAPI "Enables Sync API support" ON)
|
||||
if(WANT_SYNCAPI)
|
||||
add_definitions(-DTHREADED)
|
||||
if(WIN32)
|
||||
# Note that the generator expression ensures that `/MTd` is used when Debug
|
||||
# configurations are built.
|
||||
add_compile_options(/MT$<$<CONFIG:Debug>:d>)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# CppUnit option
|
||||
if(WIN32 OR APPLE)
|
||||
# The tests do not yet compile on Windows or macOS,
|
||||
# so we set this to off by default.
|
||||
#
|
||||
# Note that CMake does not have expressions except in conditionals,
|
||||
# so we're left with this if/else/endif pattern.
|
||||
set(DEFAULT_WANT_CPPUNIT OFF)
|
||||
else()
|
||||
set(DEFAULT_WANT_CPPUNIT ON)
|
||||
endif()
|
||||
option(WANT_CPPUNIT "Enables CppUnit and tests" ${DEFAULT_WANT_CPPUNIT})
|
||||
|
||||
# The function `to_have(in out)` converts a header name like `arpa/inet.h`
|
||||
# into an Autotools style preprocessor definition `HAVE_ARPA_INET_H`.
|
||||
# This is then set or unset in `configure_file()` step.
|
||||
#
|
||||
# Note that CMake functions do not have return values; instead an "out"
|
||||
# variable must be passed, and explicitly set with parent scope.
|
||||
function(to_have in out)
|
||||
string(TOUPPER ${in} str)
|
||||
string(REGEX REPLACE "/|\\." "_" str ${str})
|
||||
set(${out} "HAVE_${str}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# include file checks
|
||||
foreach(f generated/zookeeper.jute.h generated/zookeeper.jute.c)
|
||||
if(EXISTS "${LIBRARY_DIR}/${f}")
|
||||
to_have(${f} name)
|
||||
set(${name} 1)
|
||||
else()
|
||||
message(FATAL_ERROR
|
||||
"jute files are missing!\n"
|
||||
"Please run 'ant compile_jute' while in the ZooKeeper top level directory.")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
# header checks
|
||||
include(CheckIncludeFile)
|
||||
set(check_headers
|
||||
arpa/inet.h
|
||||
dlfcn.h
|
||||
fcntl.h
|
||||
inttypes.h
|
||||
memory.h
|
||||
netdb.h
|
||||
netinet/in.h
|
||||
stdint.h
|
||||
stdlib.h
|
||||
string.h
|
||||
strings.h
|
||||
sys/socket.h
|
||||
sys/stat.h
|
||||
sys/time.h
|
||||
sys/types.h
|
||||
unistd.h
|
||||
sys/utsname.h)
|
||||
|
||||
foreach(f ${check_headers})
|
||||
to_have(${f} name)
|
||||
check_include_file(${f} ${name})
|
||||
endforeach()
|
||||
|
||||
# function checks
|
||||
include(CheckFunctionExists)
|
||||
set(check_functions
|
||||
getcwd
|
||||
gethostbyname
|
||||
gethostname
|
||||
getlogin
|
||||
getpwuid_r
|
||||
gettimeofday
|
||||
getuid
|
||||
memmove
|
||||
memset
|
||||
poll
|
||||
socket
|
||||
strchr
|
||||
strdup
|
||||
strerror
|
||||
strtol)
|
||||
|
||||
foreach(fn ${check_functions})
|
||||
to_have(${fn} name)
|
||||
check_function_exists(${fn} ${name})
|
||||
endforeach()
|
||||
|
||||
# library checks
|
||||
set(check_libraries rt m pthread)
|
||||
foreach(lib ${check_libraries})
|
||||
to_have("lib${lib}" name)
|
||||
find_library(${name} ${lib})
|
||||
endforeach()
|
||||
|
||||
# IPv6 check
|
||||
include(CheckStructHasMember)
|
||||
check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h" ZOO_IPV6_ENABLED)
|
||||
|
||||
# configure
|
||||
configure_file(${LIBRARY_DIR}/cmake_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/config.h)
|
||||
|
||||
# hashtable library
|
||||
set(hashtable_sources ${LIBRARY_DIR}/src/hashtable/hashtable_itr.c ${LIBRARY_DIR}/src/hashtable/hashtable.c)
|
||||
add_library(hashtable STATIC ${hashtable_sources})
|
||||
|
||||
# zookeeper library
|
||||
set(zookeeper_sources
|
||||
${LIBRARY_DIR}/src/zookeeper.c
|
||||
${LIBRARY_DIR}/src/recordio.c
|
||||
${LIBRARY_DIR}/generated/zookeeper.jute.c
|
||||
${LIBRARY_DIR}/src/zk_log.c
|
||||
${LIBRARY_DIR}/src/zk_hashtable.c
|
||||
)
|
||||
# src/addrvec.c
|
||||
|
||||
if(WANT_SYNCAPI)
|
||||
list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/mt_adaptor.c)
|
||||
else()
|
||||
list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/st_adaptor.c)
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/winport.c)
|
||||
endif()
|
||||
|
||||
add_library(zookeeper STATIC ${zookeeper_sources})
|
||||
target_include_directories(zookeeper BEFORE PUBLIC ${LIBRARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/include ${LIBRARY_DIR}/generated)
|
||||
target_link_libraries(zookeeper PUBLIC
|
||||
hashtable
|
||||
$<$<PLATFORM_ID:Linux>:rt> # clock_gettime
|
||||
$<$<PLATFORM_ID:Windows>:ws2_32>) # Winsock 2.0
|
||||
|
||||
if(WANT_SYNCAPI AND NOT WIN32)
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(zookeeper PUBLIC Threads::Threads)
|
||||
endif()
|
||||
|
||||
# cli executable
|
||||
add_executable(cli ${LIBRARY_DIR}/src/cli.c)
|
||||
target_link_libraries(cli zookeeper)
|
||||
|
||||
# load_gen executable
|
||||
if(WANT_SYNCAPI AND NOT WIN32)
|
||||
add_executable(load_gen ${LIBRARY_DIR}/src/load_gen.c)
|
||||
target_link_libraries(load_gen zookeeper)
|
||||
endif()
|
||||
@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io
|
||||
${LINK_LIBRARIES_ONLY_ON_X86_64}
|
||||
${LZ4_LIBRARY}
|
||||
${ZSTD_LIBRARY}
|
||||
${ZOOKEEPER_LIBRARY}
|
||||
${DOUBLE_CONVERSION_LIBRARIES}
|
||||
${Poco_Net_LIBRARY}
|
||||
${Poco_Data_LIBRARY}
|
||||
@ -158,10 +157,6 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
|
||||
target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_BOOST_LIBRARY)
|
||||
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
endif ()
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
set(VERSION_DESCRIBE v1.1.54363-testing)
|
||||
set(VERSION_REVISION 54363)
|
||||
set(VERSION_DESCRIBE v1.1.54371-testing)
|
||||
set(VERSION_REVISION 54371)
|
||||
# end of autochange
|
||||
|
||||
set (VERSION_MAJOR 1)
|
||||
|
||||
@ -70,7 +70,10 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
|
||||
return combinator->transformAggregateFunction(nested_function, argument_types, parameters);
|
||||
}
|
||||
|
||||
return getImpl(name, argument_types, parameters, recursion_level);
|
||||
auto res = getImpl(name, argument_types, parameters, recursion_level);
|
||||
if (!res)
|
||||
throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionMaxIntersections.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
AggregateFunctionPtr createAggregateFunctionMaxIntersections(
|
||||
AggregateFunctionIntersectionsKind kind,
|
||||
const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{
|
||||
assertBinary(name, argument_types);
|
||||
assertNoParameters(name, parameters);
|
||||
|
||||
AggregateFunctionPtr res(createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types));
|
||||
if (!res)
|
||||
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
|
||||
+ " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("maxIntersections", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{ return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Count, name, argument_types, parameters); });
|
||||
|
||||
factory.registerFunction("maxIntersectionsPosition", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{ return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Position, name, argument_types, parameters); });
|
||||
}
|
||||
|
||||
}
|
||||
170
dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
Normal file
170
dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
Normal file
@ -0,0 +1,170 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <Common/ArenaAllocator.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
|
||||
#define AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE 0xFFFFFF
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int TOO_LARGE_ARRAY_SIZE;
|
||||
}
|
||||
|
||||
|
||||
/** maxIntersections: returns maximum count of the intersected intervals defined by start_column and end_column values,
|
||||
* maxIntersectionsPosition: returns leftmost position of maximum intersection of intervals.
|
||||
*/
|
||||
|
||||
/// Similar to GroupArrayNumericData.
|
||||
template <typename T>
|
||||
struct MaxIntersectionsData
|
||||
{
|
||||
/// Left or right end of the interval and signed weight; with positive sign for begin of interval and negative sign for end of interval.
|
||||
using Value = std::pair<T, Int64>;
|
||||
|
||||
// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
|
||||
using Allocator = MixedArenaAllocator<4096>;
|
||||
using Array = PODArray<Value, 32, Allocator>;
|
||||
|
||||
Array value;
|
||||
};
|
||||
|
||||
enum class AggregateFunctionIntersectionsKind
|
||||
{
|
||||
Count,
|
||||
Position
|
||||
};
|
||||
|
||||
template <typename PointType>
|
||||
class AggregateFunctionIntersectionsMax final
|
||||
: public IAggregateFunctionDataHelper<MaxIntersectionsData<PointType>, AggregateFunctionIntersectionsMax<PointType>>
|
||||
{
|
||||
private:
|
||||
AggregateFunctionIntersectionsKind kind;
|
||||
|
||||
public:
|
||||
AggregateFunctionIntersectionsMax(AggregateFunctionIntersectionsKind kind_, const DataTypes & arguments)
|
||||
: kind(kind_)
|
||||
{
|
||||
if (!arguments[0]->isNumber())
|
||||
throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
if (!arguments[1]->isNumber())
|
||||
throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
if (!arguments[0]->equals(*arguments[1]))
|
||||
throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return kind == AggregateFunctionIntersectionsKind::Count
|
||||
? "maxIntersections"
|
||||
: "maxIntersectionsPosition";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
if (kind == AggregateFunctionIntersectionsKind::Count)
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
else
|
||||
return std::make_shared<DataTypeNumber<PointType>>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
|
||||
{
|
||||
PointType left = static_cast<const ColumnVector<PointType> &>(*columns[0]).getData()[row_num];
|
||||
PointType right = static_cast<const ColumnVector<PointType> &>(*columns[1]).getData()[row_num];
|
||||
|
||||
if (!isNaN(left))
|
||||
this->data(place).value.push_back(std::make_pair(left, Int64(1)), arena);
|
||||
|
||||
if (!isNaN(right))
|
||||
this->data(place).value.push_back(std::make_pair(right, Int64(-1)), arena);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
|
||||
{
|
||||
auto & cur_elems = this->data(place);
|
||||
auto & rhs_elems = this->data(rhs);
|
||||
|
||||
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
const auto & value = this->data(place).value;
|
||||
size_t size = value.size();
|
||||
writeVarUInt(size, buf);
|
||||
buf.write(reinterpret_cast<const char *>(&value[0]), size * sizeof(value[0]));
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
|
||||
{
|
||||
size_t size = 0;
|
||||
readVarUInt(size, buf);
|
||||
|
||||
if (unlikely(size > AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE))
|
||||
throw Exception("Too large array size", ErrorCodes::TOO_LARGE_ARRAY_SIZE);
|
||||
|
||||
auto & value = this->data(place).value;
|
||||
|
||||
value.resize(size, arena);
|
||||
buf.read(reinterpret_cast<char *>(&value[0]), size * sizeof(value[0]));
|
||||
}
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
Int64 current_intersections = 0;
|
||||
Int64 max_intersections = 0;
|
||||
PointType position_of_max_intersections = 0;
|
||||
|
||||
/// const_cast because we will sort the array
|
||||
auto & array = const_cast<typename MaxIntersectionsData<PointType>::Array &>(this->data(place).value);
|
||||
|
||||
std::sort(array.begin(), array.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
|
||||
for (const auto & point_weight : array)
|
||||
{
|
||||
current_intersections += point_weight.second;
|
||||
if (current_intersections > max_intersections)
|
||||
{
|
||||
max_intersections = current_intersections;
|
||||
position_of_max_intersections = point_weight.first;
|
||||
}
|
||||
}
|
||||
|
||||
if (kind == AggregateFunctionIntersectionsKind::Count)
|
||||
{
|
||||
auto & result_column = static_cast<ColumnUInt64 &>(to).getData();
|
||||
result_column.push_back(max_intersections);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto & result_column = static_cast<ColumnVector<PointType> &>(to).getData();
|
||||
result_column.push_back(position_of_max_intersections);
|
||||
}
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override
|
||||
{
|
||||
return __FILE__;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -37,7 +37,11 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con
|
||||
values_types.push_back(array_type->getNestedType());
|
||||
}
|
||||
|
||||
return AggregateFunctionPtr(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
|
||||
AggregateFunctionPtr res(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
|
||||
if (!res)
|
||||
throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,6 +33,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
{
|
||||
assertNoParameters(name, params);
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -51,23 +55,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data>>();
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
|
||||
template <template <typename> class Data, typename DataForVariadic>
|
||||
@ -75,6 +74,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
{
|
||||
assertNoParameters(name, params);
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -93,23 +96,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data<DataTypeUUID::FieldType>>>();
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -122,8 +122,8 @@ struct AggregateFunctionUniqExactData<String>
|
||||
static String getName() { return "uniqExact"; }
|
||||
};
|
||||
|
||||
template <typename T, HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
{
|
||||
using Key = UInt32;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
@ -135,14 +135,15 @@ struct BaseUniqCombinedData
|
||||
TrivialHash,
|
||||
UInt32,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
template <HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData<String, mode>
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedData<String>
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
@ -154,44 +155,14 @@ struct BaseUniqCombinedData<String, mode>
|
||||
TrivialHash,
|
||||
UInt64,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
HyperLogLogMode::FullFeatured>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
/// Aggregate functions uniqCombinedRaw, uniqCombinedLinearCounting, and uniqCombinedBiasCorrected
|
||||
/// are intended for development of new versions of the uniqCombined function.
|
||||
/// Users should only use uniqCombined.
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedRawData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::Raw>
|
||||
{
|
||||
static String getName() { return "uniqCombinedRaw"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedLinearCountingData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::LinearCounting>
|
||||
{
|
||||
static String getName() { return "uniqCombinedLinearCounting"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedBiasCorrectedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::BiasCorrected>
|
||||
{
|
||||
static String getName() { return "uniqCombinedBiasCorrected"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::FullFeatured>
|
||||
{
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
@ -288,10 +259,7 @@ struct OneAdder
|
||||
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedRawData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedLinearCountingData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedBiasCorrectedData<T>>
|
||||
|| std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
|
||||
{
|
||||
if constexpr (!std::is_same_v<T, String>)
|
||||
{
|
||||
@ -371,7 +339,7 @@ public:
|
||||
|
||||
/** For multiple arguments. To compute, hashes them.
|
||||
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
|
||||
* But (for the possibility of effective implementation), you can not pass several arguments, among which there are tuples.
|
||||
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
|
||||
*/
|
||||
template <typename Data, bool argument_is_tuple>
|
||||
class AggregateFunctionUniqVariadic final : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniqVariadic<Data, argument_is_tuple>>
|
||||
|
||||
@ -42,6 +42,10 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
|
||||
threshold = threshold_param;
|
||||
}
|
||||
|
||||
if (argument_types.empty())
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (argument_types.size() == 1)
|
||||
{
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
@ -60,22 +64,18 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<true>>(argument_types, threshold);
|
||||
else if (typeid_cast<const DataTypeUUID *>(&argument_type))
|
||||
return std::make_shared<AggregateFunctionUniqUpTo<DataTypeUUID::FieldType>>(threshold);
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (argument_types.size() > 1)
|
||||
else
|
||||
{
|
||||
/// If there are several arguments, then no tuples allowed among them.
|
||||
for (const auto & type : argument_types)
|
||||
if (typeid_cast<const DataTypeTuple *>(type.get()))
|
||||
throw Exception("Tuple argument of function " + name + " must be the only argument",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
|
||||
}
|
||||
else
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name,
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
/// "Variadic" method also works as a fallback generic case for single argument.
|
||||
return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <Core/Types.h>
|
||||
@ -32,7 +33,9 @@ struct QuantileExact
|
||||
|
||||
void add(const Value & x)
|
||||
{
|
||||
array.push_back(x);
|
||||
/// We must skip NaNs as they are not compatible with comparison sorting.
|
||||
if (!isNaN(x))
|
||||
array.push_back(x);
|
||||
}
|
||||
|
||||
template <typename Weight>
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -33,12 +34,15 @@ struct QuantileExactWeighted
|
||||
|
||||
void add(const Value & x)
|
||||
{
|
||||
++map[x];
|
||||
/// We must skip NaNs as they are not compatible with comparison sorting.
|
||||
if (!isNaN(x))
|
||||
++map[x];
|
||||
}
|
||||
|
||||
void add(const Value & x, const Weight & weight)
|
||||
{
|
||||
map[x] += weight;
|
||||
if (!isNaN(x))
|
||||
map[x] += weight;
|
||||
}
|
||||
|
||||
void merge(const QuantileExactWeighted & rhs)
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -12,6 +13,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int PARAMETER_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
|
||||
@ -55,6 +57,10 @@ struct QuantileLevels
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
levels[i] = applyVisitor(FieldVisitorConvertToNumber<Float64>(), params[i]);
|
||||
|
||||
if (isNaN(levels[i]) || levels[i] < 0 || levels[i] > 1)
|
||||
throw Exception("Quantile level is out of range [0..1]", ErrorCodes::PARAMETER_OUT_OF_BOUND);
|
||||
|
||||
permutation[i] = i;
|
||||
}
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <Poco/Exception.h>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
@ -67,6 +68,9 @@ public:
|
||||
|
||||
void insert(const T & v)
|
||||
{
|
||||
if (isNaN(v))
|
||||
return;
|
||||
|
||||
sorted = false;
|
||||
++total_values;
|
||||
if (samples.size() < sample_count)
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <Poco/Exception.h>
|
||||
|
||||
|
||||
@ -66,6 +67,9 @@ public:
|
||||
|
||||
void insert(const T & v, const UInt64 determinator)
|
||||
{
|
||||
if (isNaN(v))
|
||||
return;
|
||||
|
||||
const UInt32 hash = intHash64(determinator);
|
||||
if (!good(hash))
|
||||
return;
|
||||
|
||||
@ -23,6 +23,7 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory &);
|
||||
|
||||
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
|
||||
void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
|
||||
@ -53,6 +54,7 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionUniqUpTo(factory);
|
||||
registerAggregateFunctionTopK(factory);
|
||||
registerAggregateFunctionsBitwise(factory);
|
||||
registerAggregateFunctionsMaxIntersections(factory);
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT
|
||||
void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
|
||||
{
|
||||
if (table.storage)
|
||||
for (const auto & name : table.storage->getColumnNamesList())
|
||||
for (const auto & name : table.storage->getColumns().getNamesOfPhysical())
|
||||
res.emplace_back(createASTIdentifierForColumnInTable(name, table));
|
||||
else
|
||||
for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
|
||||
@ -315,7 +315,7 @@ void processIdentifier(
|
||||
}
|
||||
else if (table->storage)
|
||||
{
|
||||
info.data_type = table->storage->getDataTypeByName(column_name);
|
||||
info.data_type = table->storage->getColumn(column_name).type;
|
||||
}
|
||||
else
|
||||
throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
|
||||
{
|
||||
writeString(table.second->getName(), out);
|
||||
writeCString("\n\n", out);
|
||||
writeString(table.second->getColumnsList().toString(), out);
|
||||
writeString(table.second->getColumns().getAllPhysical().toString(), out);
|
||||
writeCString("\n", out);
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <DataStreams/NativeBlockInputStream.h>
|
||||
#include <DataStreams/NativeBlockOutputStream.h>
|
||||
#include <Client/Connection.h>
|
||||
#include <Client/TimeoutSetter.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/NetException.h>
|
||||
@ -50,14 +51,15 @@ void Connection::connect()
|
||||
if (connected)
|
||||
disconnect();
|
||||
|
||||
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user);
|
||||
LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
|
||||
<< (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
|
||||
|
||||
if (static_cast<bool>(encryption))
|
||||
if (static_cast<bool>(secure))
|
||||
{
|
||||
#if Poco_NetSSL_FOUND
|
||||
socket = std::make_unique<Poco::Net::SecureStreamSocket>();
|
||||
#else
|
||||
throw Exception{"tcp_ssl protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
throw Exception{"tcp_secure protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
}
|
||||
else
|
||||
@ -148,6 +150,10 @@ void Connection::receiveHello()
|
||||
{
|
||||
readStringBinary(server_timezone, *in);
|
||||
}
|
||||
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
|
||||
{
|
||||
readStringBinary(server_display_name, *in);
|
||||
}
|
||||
}
|
||||
else if (packet_type == Protocol::Server::Exception)
|
||||
receiveException()->rethrow();
|
||||
@ -203,6 +209,14 @@ const String & Connection::getServerTimezone()
|
||||
return server_timezone;
|
||||
}
|
||||
|
||||
const String & Connection::getServerDisplayName()
|
||||
{
|
||||
if (!connected)
|
||||
connect();
|
||||
|
||||
return server_display_name;
|
||||
}
|
||||
|
||||
void Connection::forceConnected()
|
||||
{
|
||||
if (!connected)
|
||||
@ -216,32 +230,6 @@ void Connection::forceConnected()
|
||||
}
|
||||
}
|
||||
|
||||
struct TimeoutSetter
|
||||
{
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
|
||||
: socket(socket_), timeout(timeout_)
|
||||
{
|
||||
old_send_timeout = socket.getSendTimeout();
|
||||
old_receive_timeout = socket.getReceiveTimeout();
|
||||
|
||||
if (old_send_timeout > timeout)
|
||||
socket.setSendTimeout(timeout);
|
||||
if (old_receive_timeout > timeout)
|
||||
socket.setReceiveTimeout(timeout);
|
||||
}
|
||||
|
||||
~TimeoutSetter()
|
||||
{
|
||||
socket.setSendTimeout(old_send_timeout);
|
||||
socket.setReceiveTimeout(old_receive_timeout);
|
||||
}
|
||||
|
||||
Poco::Net::StreamSocket & socket;
|
||||
Poco::Timespan timeout;
|
||||
Poco::Timespan old_send_timeout;
|
||||
Poco::Timespan old_receive_timeout;
|
||||
};
|
||||
|
||||
bool Connection::ping()
|
||||
{
|
||||
// LOG_TRACE(log_wrapper.get(), "Ping");
|
||||
|
||||
@ -58,14 +58,14 @@ public:
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
|
||||
:
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(host, port),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
encryption(encryption_),
|
||||
secure(secure_),
|
||||
timeouts(timeouts_),
|
||||
sync_request_timeout(sync_request_timeout_),
|
||||
log_wrapper(*this)
|
||||
@ -84,7 +84,7 @@ public:
|
||||
const ConnectionTimeouts & timeouts_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
|
||||
:
|
||||
host(host_), port(port_),
|
||||
@ -93,7 +93,7 @@ public:
|
||||
resolved_address(resolved_address_),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
encryption(encryption_),
|
||||
secure(secure_),
|
||||
timeouts(timeouts_),
|
||||
sync_request_timeout(sync_request_timeout_),
|
||||
log_wrapper(*this)
|
||||
@ -134,6 +134,7 @@ public:
|
||||
void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
|
||||
|
||||
const String & getServerTimezone();
|
||||
const String & getServerDisplayName();
|
||||
|
||||
/// For log and exception messages.
|
||||
const String & getDescription() const;
|
||||
@ -213,6 +214,7 @@ private:
|
||||
UInt64 server_version_minor = 0;
|
||||
UInt64 server_revision = 0;
|
||||
String server_timezone;
|
||||
String server_display_name;
|
||||
|
||||
std::unique_ptr<Poco::Net::StreamSocket> socket;
|
||||
std::shared_ptr<ReadBuffer> in;
|
||||
@ -220,7 +222,7 @@ private:
|
||||
|
||||
String query_id;
|
||||
Protocol::Compression compression; /// Enable data compression for communication.
|
||||
Protocol::Encryption encryption; /// Enable data encryption for communication.
|
||||
Protocol::Secure secure; /// Enable data encryption for communication.
|
||||
|
||||
/// What compression settings to use while sending data for INSERT queries and external tables.
|
||||
CompressionSettings compression_settings;
|
||||
|
||||
@ -51,11 +51,12 @@ public:
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable)
|
||||
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(host_, port_),
|
||||
client_name(client_name_), compression(compression_), encryption(encryption_),
|
||||
client_name(client_name_), compression(compression_),
|
||||
secure{secure_},
|
||||
timeouts(timeouts)
|
||||
{
|
||||
}
|
||||
@ -67,11 +68,12 @@ public:
|
||||
const ConnectionTimeouts & timeouts,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable)
|
||||
: Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_), resolved_address(resolved_address_),
|
||||
client_name(client_name_), compression(compression_), encryption(encryption_),
|
||||
client_name(client_name_), compression(compression_),
|
||||
secure{secure_},
|
||||
timeouts(timeouts)
|
||||
{
|
||||
}
|
||||
@ -102,7 +104,7 @@ protected:
|
||||
return std::make_shared<Connection>(
|
||||
host, port, resolved_address,
|
||||
default_database, user, password, timeouts,
|
||||
client_name, compression, encryption);
|
||||
client_name, compression, secure);
|
||||
}
|
||||
|
||||
private:
|
||||
@ -119,7 +121,7 @@ private:
|
||||
|
||||
String client_name;
|
||||
Protocol::Compression compression; /// Whether to compress data when interacting with the server.
|
||||
Protocol::Encryption encryption; /// Whether to encrypt data when interacting with the server.
|
||||
Protocol::Secure secure; /// Whether to encrypt data when interacting with the server.
|
||||
|
||||
ConnectionTimeouts timeouts;
|
||||
};
|
||||
|
||||
46
dbms/src/Client/TimeoutSetter.h
Normal file
46
dbms/src/Client/TimeoutSetter.h
Normal file
@ -0,0 +1,46 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Timespan.h>
|
||||
#include <Poco/Net/StreamSocket.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Temporarily overrides socket send/recieve timeouts and reset them back into destructor
|
||||
/// Timeouts could be only decreased
|
||||
struct TimeoutSetter
|
||||
{
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & send_timeout_, const Poco::Timespan & recieve_timeout_)
|
||||
: socket(socket_), send_timeout(send_timeout_), recieve_timeout(recieve_timeout_)
|
||||
{
|
||||
old_send_timeout = socket.getSendTimeout();
|
||||
old_receive_timeout = socket.getReceiveTimeout();
|
||||
|
||||
if (old_send_timeout > send_timeout)
|
||||
socket.setSendTimeout(send_timeout);
|
||||
|
||||
if (old_receive_timeout > recieve_timeout)
|
||||
socket.setReceiveTimeout(recieve_timeout);
|
||||
}
|
||||
|
||||
TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
|
||||
: TimeoutSetter(socket_, timeout_, timeout_) {}
|
||||
|
||||
~TimeoutSetter()
|
||||
{
|
||||
socket.setSendTimeout(old_send_timeout);
|
||||
socket.setReceiveTimeout(old_receive_timeout);
|
||||
}
|
||||
|
||||
Poco::Net::StreamSocket & socket;
|
||||
|
||||
Poco::Timespan send_timeout;
|
||||
Poco::Timespan recieve_timeout;
|
||||
|
||||
Poco::Timespan old_send_timeout;
|
||||
Poco::Timespan old_receive_timeout;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = getData().size();
|
||||
if (size != filter.size())
|
||||
@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = getData().size();
|
||||
|
||||
@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n)
|
||||
data.resize_assume_reserved(new_size);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
|
||||
ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != offsets.size())
|
||||
|
||||
@ -152,11 +152,11 @@ public:
|
||||
|
||||
void popBack(size_t n) override;
|
||||
|
||||
MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||
|
||||
|
||||
@ -31,10 +31,10 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
|
||||
: data(nested_column), offsets(offsets_column)
|
||||
ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
|
||||
: data(std::move(nested_column)), offsets(std::move(offsets_column))
|
||||
{
|
||||
if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
|
||||
if (!typeid_cast<const ColumnOffsets *>(offsets.get()))
|
||||
throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
/** NOTE
|
||||
@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs
|
||||
*/
|
||||
}
|
||||
|
||||
ColumnArray::ColumnArray(const ColumnPtr & nested_column)
|
||||
: data(nested_column)
|
||||
ColumnArray::ColumnArray(MutableColumnPtr && nested_column)
|
||||
: data(std::move(nested_column))
|
||||
{
|
||||
if (!data->empty())
|
||||
throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN);
|
||||
@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const
|
||||
ColumnPtr ColumnArray::convertToFullColumnIfConst() const
|
||||
{
|
||||
ColumnPtr new_data;
|
||||
|
||||
@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return filterNumber<UInt8>(filt, result_size_hint);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return filterNumber<UInt16>(filt, result_size_hint);
|
||||
@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t col_size = getOffsets().size();
|
||||
if (col_size != filt.size())
|
||||
@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = getOffsets().size();
|
||||
if (size != filt.size())
|
||||
@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result
|
||||
filtered_offsets);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (getOffsets().size() == 0)
|
||||
return ColumnArray::create(data);
|
||||
@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
|
||||
|
||||
Columns temporary_arrays(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint);
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.filter(filt, result_size_hint);
|
||||
|
||||
Columns tuple_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = getOffsets().size();
|
||||
|
||||
@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
{
|
||||
if (typeid_cast<const ColumnUInt8 *>(data.get())) return replicateNumber<UInt8>(replicate_offsets);
|
||||
if (typeid_cast<const ColumnUInt16 *>(data.get())) return replicateNumber<UInt16>(replicate_offsets);
|
||||
@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
|
||||
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets)
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
|
||||
{
|
||||
const ColumnNullable & nullable = static_cast<const ColumnNullable &>(*data);
|
||||
|
||||
/// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result.
|
||||
/// NOTE Offsets are calculated twice and it is redundant.
|
||||
|
||||
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
|
||||
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
|
||||
auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
|
||||
return ColumnArray::create(
|
||||
ColumnNullable::create(
|
||||
static_cast<ColumnArray &>(*array_of_nested).getDataPtr(),
|
||||
static_cast<ColumnArray &>(*array_of_null_map).getDataPtr()),
|
||||
static_cast<ColumnArray &>(*array_of_nested).getOffsetsPtr());
|
||||
static_cast<const ColumnArray &>(*array_of_nested).getDataPtr(),
|
||||
static_cast<const ColumnArray &>(*array_of_null_map).getDataPtr()),
|
||||
static_cast<const ColumnArray &>(*array_of_nested).getOffsetsPtr());
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
|
||||
{
|
||||
const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
|
||||
|
||||
@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets)
|
||||
|
||||
Columns temporary_arrays(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets);
|
||||
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
|
||||
.replicate(replicate_offsets);
|
||||
|
||||
Columns tuple_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
|
||||
@ -24,14 +24,32 @@ private:
|
||||
friend class COWPtrHelper<IColumn, ColumnArray>;
|
||||
|
||||
/** Create an array column with specified values and offsets. */
|
||||
ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column);
|
||||
ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
|
||||
|
||||
/** Create an empty column of arrays with the type of values as in the column `nested_column` */
|
||||
ColumnArray(const ColumnPtr & nested_column);
|
||||
explicit ColumnArray(MutableColumnPtr && nested_column);
|
||||
|
||||
ColumnArray(const ColumnArray &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnArray>;
|
||||
|
||||
static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
|
||||
{
|
||||
return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
|
||||
}
|
||||
|
||||
static Ptr create(const ColumnPtr & nested_column)
|
||||
{
|
||||
return ColumnArray::create(nested_column->assumeMutable());
|
||||
}
|
||||
|
||||
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
|
||||
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
|
||||
|
||||
/** On the index i there is an offset to the beginning of the i + 1 -th element. */
|
||||
using ColumnOffsets = ColumnVector<Offset>;
|
||||
|
||||
@ -51,15 +69,15 @@ public:
|
||||
void insertFrom(const IColumn & src_, size_t n) override;
|
||||
void insertDefault() override;
|
||||
void popBack(size_t n) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
|
||||
void reserve(size_t n) override;
|
||||
size_t byteSize() const override;
|
||||
size_t allocatedBytes() const override;
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
MutableColumnPtr convertToFullColumnIfConst() const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr convertToFullColumnIfConst() const override;
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
bool hasEqualOffsets(const ColumnArray & other) const;
|
||||
@ -110,33 +128,33 @@ private:
|
||||
|
||||
/// Multiply values if the nested column is ColumnVector<T>.
|
||||
template <typename T>
|
||||
MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
|
||||
|
||||
/// Multiply the values if the nested column is ColumnString. The code is too complicated.
|
||||
MutableColumnPtr replicateString(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateString(const Offsets & replicate_offsets) const;
|
||||
|
||||
/** Non-constant arrays of constant values are quite rare.
|
||||
* Most functions can not work with them, and does not create such columns as a result.
|
||||
* An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions.
|
||||
* Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst).
|
||||
*/
|
||||
MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateConst(const Offsets & replicate_offsets) const;
|
||||
|
||||
/** The following is done by simply replicating of nested columns.
|
||||
*/
|
||||
MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
|
||||
MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
|
||||
MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
|
||||
ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
|
||||
|
||||
|
||||
/// Specializations for the filter function.
|
||||
template <typename T>
|
||||
MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
|
||||
|
||||
MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
|
||||
MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
|
||||
ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s)
|
||||
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::convertToFullColumn() const
|
||||
ColumnPtr ColumnConst::convertToFullColumn() const
|
||||
{
|
||||
return data->replicate(Offsets(1, s));
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
|
||||
ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
|
||||
{
|
||||
if (s != filt.size())
|
||||
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")",
|
||||
@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_
|
||||
return ColumnConst::create(data, countBytesInFilter(filt));
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
{
|
||||
if (s != offsets.size())
|
||||
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")",
|
||||
@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
|
||||
return ColumnConst::create(data, replicated_size);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
if (limit == 0)
|
||||
limit = s;
|
||||
|
||||
@ -29,9 +29,9 @@ private:
|
||||
ColumnConst(const ColumnConst & src) = default;
|
||||
|
||||
public:
|
||||
MutableColumnPtr convertToFullColumn() const;
|
||||
ColumnPtr convertToFullColumn() const;
|
||||
|
||||
MutableColumnPtr convertToFullColumnIfConst() const override
|
||||
ColumnPtr convertToFullColumnIfConst() const override
|
||||
{
|
||||
return convertToFullColumn();
|
||||
}
|
||||
@ -145,9 +145,9 @@ public:
|
||||
data->updateHashWithValue(0, hash);
|
||||
}
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
|
||||
|
||||
size_t byteSize() const override
|
||||
|
||||
@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
|
||||
memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != filt.size())
|
||||
@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
|
||||
@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != offsets.size())
|
||||
|
||||
@ -104,11 +104,11 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
|
||||
{
|
||||
|
||||
@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
|
||||
return ColumnFunction::create(size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
{
|
||||
if (size_ != offsets.size())
|
||||
throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column ("
|
||||
@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
|
||||
return ColumnFunction::create(replicated_size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
{
|
||||
ColumnsWithTypeAndName capture = captured_columns;
|
||||
for (auto & column : capture)
|
||||
@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
|
||||
return ColumnFunction::create(length, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (size_ != filt.size())
|
||||
throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column ("
|
||||
@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size
|
||||
return ColumnFunction::create(filtered_size, function, capture);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
if (limit == 0)
|
||||
limit = size_;
|
||||
|
||||
@ -29,10 +29,10 @@ public:
|
||||
|
||||
size_t size() const override { return size_; }
|
||||
|
||||
MutableColumnPtr cut(size_t start, size_t length) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr cut(size_t start, size_t length) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
void insertDefault() override;
|
||||
void popBack(size_t n) override;
|
||||
std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex num_columns,
|
||||
|
||||
@ -18,8 +18,8 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
|
||||
: nested_column{nested_column_}, null_map{null_map_}
|
||||
ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_)
|
||||
: nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
|
||||
{
|
||||
/// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
|
||||
if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
|
||||
@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
|
||||
|
||||
MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
|
||||
{
|
||||
ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
|
||||
MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
|
||||
auto new_null_map = ColumnUInt8::create();
|
||||
|
||||
if (new_size > 0)
|
||||
@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
|
||||
memset(&new_null_map->getData()[count], 1, new_size - count);
|
||||
}
|
||||
|
||||
return ColumnNullable::create(new_nested_col, std::move(new_null_map));
|
||||
return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map));
|
||||
}
|
||||
|
||||
|
||||
@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n)
|
||||
getNullMapColumn().popBack(n);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint);
|
||||
ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint);
|
||||
return ColumnNullable::create(filtered_data, filtered_null_map);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
ColumnPtr permuted_data = getNestedColumn().permute(perm, limit);
|
||||
ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit);
|
||||
@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
|
||||
{
|
||||
ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
|
||||
ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);
|
||||
|
||||
@ -23,10 +23,22 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
|
||||
private:
|
||||
friend class COWPtrHelper<IColumn, ColumnNullable>;
|
||||
|
||||
ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_);
|
||||
ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
|
||||
ColumnNullable(const ColumnNullable &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnNullable>;
|
||||
static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
|
||||
{
|
||||
return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
|
||||
}
|
||||
|
||||
template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
|
||||
static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
|
||||
|
||||
const char * getFamilyName() const override { return "Nullable"; }
|
||||
std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
|
||||
MutableColumnPtr cloneResized(size_t size) const override;
|
||||
@ -50,14 +62,14 @@ public:
|
||||
}
|
||||
|
||||
void popBack(size_t n) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
|
||||
void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
|
||||
void reserve(size_t n) override;
|
||||
size_t byteSize() const override;
|
||||
size_t allocatedBytes() const override;
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
void updateHashWithValue(size_t n, SipHash & hash) const override;
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
|
||||
@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
if (offsets.size() == 0)
|
||||
return ColumnString::create();
|
||||
@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = offsets.size();
|
||||
|
||||
@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
|
||||
ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
|
||||
{
|
||||
size_t col_size = size();
|
||||
if (col_size != replicate_offsets.size())
|
||||
|
||||
@ -89,6 +89,12 @@ public:
|
||||
return StringRef(&chars[offsetAt(n)], sizeAt(n));
|
||||
}
|
||||
|
||||
/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
|
||||
#endif
|
||||
|
||||
void insert(const Field & x) override
|
||||
{
|
||||
const String & s = DB::get<const String &>(x);
|
||||
@ -101,7 +107,11 @@ public:
|
||||
offsets.push_back(new_size);
|
||||
}
|
||||
|
||||
void insertFrom(const IColumn & src_, size_t n) override
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
void insertFrom(const IColumn & src_, size_t n) override
|
||||
{
|
||||
const ColumnString & src = static_cast<const ColumnString &>(src_);
|
||||
|
||||
@ -206,9 +216,9 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
|
||||
void insertDefault() override
|
||||
{
|
||||
@ -239,7 +249,7 @@ public:
|
||||
/// Sorting with respect of collation.
|
||||
void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
|
||||
{
|
||||
|
||||
@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const
|
||||
return res.str();
|
||||
}
|
||||
|
||||
ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns)
|
||||
ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
|
||||
{
|
||||
columns.reserve(mutable_columns.size());
|
||||
for (auto & column : mutable_columns)
|
||||
{
|
||||
if (column->isColumnConst())
|
||||
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
|
||||
|
||||
columns.push_back(std::move(column));
|
||||
}
|
||||
}
|
||||
|
||||
ColumnTuple::Ptr ColumnTuple::create(const Columns & columns)
|
||||
{
|
||||
for (const auto & column : columns)
|
||||
if (column->isColumnConst())
|
||||
throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
|
||||
|
||||
auto column_tuple = ColumnTuple::create(MutableColumns());
|
||||
column_tuple->columns = columns;
|
||||
|
||||
return std::move(column_tuple);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::cloneEmpty() const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
MutableColumns new_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
new_columns[i] = columns[i]->cloneEmpty();
|
||||
|
||||
return ColumnTuple::create(new_columns);
|
||||
return ColumnTuple::create(std::move(new_columns));
|
||||
}
|
||||
|
||||
Field ColumnTuple::operator[](size_t n) const
|
||||
@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng
|
||||
start, length);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi
|
||||
return ColumnTuple::create(new_columns);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co
|
||||
return ColumnTuple::create(new_columns);
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
|
||||
ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
|
||||
{
|
||||
const size_t tuple_size = columns.size();
|
||||
Columns new_columns(tuple_size);
|
||||
@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se
|
||||
|
||||
for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
|
||||
{
|
||||
Columns new_columns(tuple_size);
|
||||
MutableColumns new_columns(tuple_size);
|
||||
for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx)
|
||||
new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
|
||||
res[scattered_idx] = ColumnTuple::create(new_columns);
|
||||
res[scattered_idx] = ColumnTuple::create(std::move(new_columns));
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@ -22,10 +22,19 @@ private:
|
||||
template <bool positive>
|
||||
struct Less;
|
||||
|
||||
ColumnTuple(const Columns & columns);
|
||||
explicit ColumnTuple(MutableColumns && columns);
|
||||
ColumnTuple(const ColumnTuple &) = default;
|
||||
|
||||
public:
|
||||
/** Create immutable column using immutable arguments. This arguments may be shared with other columns.
|
||||
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
|
||||
*/
|
||||
using Base = COWPtrHelper<IColumn, ColumnTuple>;
|
||||
static Ptr create(const Columns & columns);
|
||||
|
||||
template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
|
||||
static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
|
||||
|
||||
std::string getName() const override;
|
||||
const char * getFamilyName() const override { return "Tuple"; }
|
||||
|
||||
@ -49,9 +58,9 @@ public:
|
||||
const char * deserializeAndInsertFromArena(const char * pos) override;
|
||||
void updateHashWithValue(size_t n, SipHash & hash) const override;
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override;
|
||||
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr replicate(const Offsets & offsets) const override;
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||
void gather(ColumnGathererStream & gatherer_stream) override;
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
|
||||
|
||||
@ -146,7 +146,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != filt.size())
|
||||
@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t r
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
|
||||
ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
|
||||
@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, siz
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
ColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
if (size != offsets.size())
|
||||
|
||||
@ -243,11 +243,11 @@ public:
|
||||
|
||||
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
|
||||
|
||||
MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
|
||||
|
||||
MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
|
||||
ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
|
||||
|
||||
MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override;
|
||||
ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
|
||||
|
||||
void getExtremes(Field & min, Field & max) const override;
|
||||
|
||||
|
||||
@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column)
|
||||
|
||||
if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(&column))
|
||||
{
|
||||
MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate();
|
||||
ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
|
||||
MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate();
|
||||
|
||||
ColumnUInt8 * concrete_column = typeid_cast<ColumnUInt8 *>(mutable_holder.get());
|
||||
if (!concrete_column)
|
||||
|
||||
@ -45,7 +45,7 @@ public:
|
||||
/** If column isn't constant, returns nullptr (or itself).
|
||||
* If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
|
||||
*/
|
||||
virtual MutablePtr convertToFullColumnIfConst() const { return {}; }
|
||||
virtual Ptr convertToFullColumnIfConst() const { return {}; }
|
||||
|
||||
/// Creates empty column with the same type.
|
||||
virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
|
||||
@ -104,11 +104,11 @@ public:
|
||||
|
||||
/// Removes all elements outside of specified range.
|
||||
/// Is used in LIMIT operation, for example.
|
||||
virtual MutablePtr cut(size_t start, size_t length) const
|
||||
virtual Ptr cut(size_t start, size_t length) const
|
||||
{
|
||||
MutablePtr res = cloneEmpty();
|
||||
res->insertRangeFrom(*this, start, length);
|
||||
return res;
|
||||
return std::move(res);
|
||||
}
|
||||
|
||||
/// Appends new value at the end of column (column's size is increased by 1).
|
||||
@ -171,12 +171,12 @@ public:
|
||||
* otherwise (i.e. < 0), makes reserve() using size of source column.
|
||||
*/
|
||||
using Filter = PaddedPODArray<UInt8>;
|
||||
virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
|
||||
virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
|
||||
|
||||
/// Permutes elements using specified permutation. Is used in sortings.
|
||||
/// limit - if it isn't 0, puts only first limit elements in the result.
|
||||
using Permutation = PaddedPODArray<size_t>;
|
||||
virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0;
|
||||
virtual Ptr permute(const Permutation & perm, size_t limit) const = 0;
|
||||
|
||||
/** Compares (*this)[n] and rhs[m].
|
||||
* Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively.
|
||||
@ -205,7 +205,7 @@ public:
|
||||
*/
|
||||
using Offset = UInt64;
|
||||
using Offsets = PaddedPODArray<Offset>;
|
||||
virtual MutablePtr replicate(const Offsets & offsets) const = 0;
|
||||
virtual Ptr replicate(const Offsets & offsets) const = 0;
|
||||
|
||||
/** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
|
||||
* Selector must contain values from 0 to num_columns - 1.
|
||||
@ -247,10 +247,10 @@ public:
|
||||
virtual void forEachSubcolumn(ColumnCallback) {}
|
||||
|
||||
|
||||
MutablePtr mutate() const
|
||||
MutablePtr mutate() const &&
|
||||
{
|
||||
MutablePtr res = COWPtr<IColumn>::mutate();
|
||||
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); });
|
||||
res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); });
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -360,4 +360,16 @@ using MutableColumns = std::vector<MutableColumnPtr>;
|
||||
using ColumnRawPtrs = std::vector<const IColumn *>;
|
||||
//using MutableColumnRawPtrs = std::vector<IColumn *>;
|
||||
|
||||
template <typename ... Args>
|
||||
struct IsMutableColumns;
|
||||
|
||||
template <typename Arg, typename ... Args>
|
||||
struct IsMutableColumns<Arg, Args ...>
|
||||
{
|
||||
static const bool value = std::is_assignable<MutableColumnPtr &&, Arg>::value && IsMutableColumns<Args ...>::value;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct IsMutableColumns<> { static const bool value = true; };
|
||||
|
||||
}
|
||||
|
||||
@ -74,12 +74,12 @@ public:
|
||||
s += length;
|
||||
}
|
||||
|
||||
MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
|
||||
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
|
||||
{
|
||||
return cloneDummy(countBytesInFilter(filt));
|
||||
}
|
||||
|
||||
MutableColumnPtr permute(const Permutation & perm, size_t limit) const override
|
||||
ColumnPtr permute(const Permutation & perm, size_t limit) const override
|
||||
{
|
||||
if (s != perm.size())
|
||||
throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
@ -94,7 +94,7 @@ public:
|
||||
res[i] = i;
|
||||
}
|
||||
|
||||
MutableColumnPtr replicate(const Offsets & offsets) const override
|
||||
ColumnPtr replicate(const Offsets & offsets) const override
|
||||
{
|
||||
if (s != offsets.size())
|
||||
throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#if !(defined(__FreeBSD__) || defined(__APPLE__))
|
||||
#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
@ -68,10 +68,9 @@
|
||||
* of this shared state.
|
||||
*
|
||||
* Caveats:
|
||||
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere
|
||||
* and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified.
|
||||
* - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere.
|
||||
* - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better
|
||||
* to use std::unique_ptr for it, but see above.
|
||||
* to use std::unique_ptr for it somehow.
|
||||
*/
|
||||
template <typename Derived>
|
||||
class COWPtr : public boost::intrusive_ref_counter<Derived>
|
||||
@ -80,12 +79,22 @@ private:
|
||||
Derived * derived() { return static_cast<Derived *>(this); }
|
||||
const Derived * derived() const { return static_cast<const Derived *>(this); }
|
||||
|
||||
template <typename T>
|
||||
class IntrusivePtr : public boost::intrusive_ptr<T>
|
||||
{
|
||||
public:
|
||||
using boost::intrusive_ptr<T>::intrusive_ptr;
|
||||
|
||||
T & operator*() const & { return boost::intrusive_ptr<T>::operator*(); }
|
||||
T && operator*() const && { return const_cast<typename std::remove_const<T>::type &&>(*boost::intrusive_ptr<T>::get()); }
|
||||
};
|
||||
|
||||
protected:
|
||||
template <typename T>
|
||||
class mutable_ptr : public boost::intrusive_ptr<T>
|
||||
class mutable_ptr : public IntrusivePtr<T>
|
||||
{
|
||||
private:
|
||||
using Base = boost::intrusive_ptr<T>;
|
||||
using Base = IntrusivePtr<T>;
|
||||
|
||||
template <typename> friend class COWPtr;
|
||||
template <typename, typename> friend class COWPtrHelper;
|
||||
@ -114,10 +123,10 @@ public:
|
||||
|
||||
protected:
|
||||
template <typename T>
|
||||
class immutable_ptr : public boost::intrusive_ptr<const T>
|
||||
class immutable_ptr : public IntrusivePtr<const T>
|
||||
{
|
||||
private:
|
||||
using Base = boost::intrusive_ptr<const T>;
|
||||
using Base = IntrusivePtr<const T>;
|
||||
|
||||
template <typename> friend class COWPtr;
|
||||
template <typename, typename> friend class COWPtrHelper;
|
||||
|
||||
@ -73,6 +73,8 @@ void ConfigReloader::run()
|
||||
|
||||
void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(reload_mutex);
|
||||
|
||||
FilesChangesTracker new_files = getNewFileList();
|
||||
if (force || new_files.isDifferOrNewerThan(files))
|
||||
{
|
||||
|
||||
@ -42,6 +42,9 @@ public:
|
||||
/// Call this method to run the backround thread.
|
||||
void start();
|
||||
|
||||
/// Reload immediately. For SYSTEM RELOAD CONFIG query.
|
||||
void reload() { reloadIfNewer(/* force */ true, /* throw_on_error */ true, /* fallback_to_preprocessed */ false); }
|
||||
|
||||
private:
|
||||
void run();
|
||||
|
||||
@ -74,6 +77,9 @@ private:
|
||||
|
||||
std::atomic<bool> quit{false};
|
||||
std::thread thread;
|
||||
|
||||
/// Locked inside reloadIfNewer.
|
||||
std::mutex reload_mutex;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -125,7 +125,6 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_SETTING = 115;
|
||||
extern const int THERE_IS_NO_DEFAULT_VALUE = 116;
|
||||
extern const int INCORRECT_DATA = 117;
|
||||
extern const int TABLE_METADATA_DOESNT_EXIST = 118;
|
||||
extern const int ENGINE_REQUIRED = 119;
|
||||
extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE = 120;
|
||||
extern const int UNKNOWN_SET_DATA_VARIANT = 121;
|
||||
@ -372,6 +371,7 @@ namespace ErrorCodes
|
||||
extern const int QUERY_WAS_CANCELLED = 394;
|
||||
extern const int FUNCTION_THROW_IF_VALUE_IS_NON_ZERO = 395;
|
||||
extern const int TOO_MANY_ROWS_OR_BYTES = 396;
|
||||
extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397;
|
||||
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
|
||||
@ -198,7 +198,7 @@ public:
|
||||
|
||||
/// Create table
|
||||
NamesAndTypesList columns = sample_block.getNamesAndTypesList();
|
||||
StoragePtr storage = StorageMemory::create(data.second, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{});
|
||||
StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
|
||||
storage->startup();
|
||||
context.addExternalTable(data.second, storage);
|
||||
BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <Common/Macros.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
@ -1,10 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <map>
|
||||
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
namespace Util
|
||||
{
|
||||
class AbstractConfiguration;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -21,10 +29,12 @@ public:
|
||||
*/
|
||||
String expand(const String & s, size_t level = 0) const;
|
||||
|
||||
private:
|
||||
using MacroMap = std::map<String, String>;
|
||||
const MacroMap getMacroMap() const { return macros; }
|
||||
|
||||
private:
|
||||
MacroMap macros;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -1,13 +1,12 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <dlfcn.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <IO/WriteBufferFromVector.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -27,6 +27,9 @@ namespace ErrorCodes
|
||||
class Throttler
|
||||
{
|
||||
public:
|
||||
Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent = nullptr)
|
||||
: max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent) {}
|
||||
|
||||
Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
|
||||
const std::shared_ptr<Throttler> & parent = nullptr)
|
||||
: max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent) {}
|
||||
@ -76,6 +79,12 @@ public:
|
||||
parent->add(amount);
|
||||
}
|
||||
|
||||
/// Not thread safe
|
||||
void setParent(const std::shared_ptr<Throttler> & parent_)
|
||||
{
|
||||
parent = parent_;
|
||||
}
|
||||
|
||||
void reset()
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
@ -3,6 +3,9 @@
|
||||
#include <Core/Types.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
|
||||
#if __SSE2__
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -49,9 +52,29 @@ inline size_t seqLength(const UInt8 first_octet)
|
||||
inline size_t countCodePoints(const UInt8 * data, size_t size)
|
||||
{
|
||||
size_t res = 0;
|
||||
const auto end = data + size;
|
||||
|
||||
/// TODO SIMD implementation looks quite simple.
|
||||
for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes.
|
||||
#if __SSE2__
|
||||
const auto bytes_sse = sizeof(__m128i);
|
||||
const auto src_end_sse = (data + size) - (size % bytes_sse);
|
||||
|
||||
const auto align_sse = _mm_set1_epi8(0x40);
|
||||
const auto upper_bound = _mm_set1_epi8(0xBF);
|
||||
|
||||
for (; data < src_end_sse; data += bytes_sse)
|
||||
{
|
||||
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
|
||||
|
||||
///Align to zero for the solve two case
|
||||
const auto align_res = _mm_adds_epu8(chars, align_sse);
|
||||
const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
|
||||
|
||||
res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
|
||||
res += (*data <= 0x7F || *data >= 0xC0);
|
||||
|
||||
return res;
|
||||
|
||||
@ -5,7 +5,11 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
|
||||
|
||||
add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
|
||||
|
||||
target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
|
||||
if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
|
||||
target_include_directories (clickhouse_common_zookeeper BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io ${ZOOKEEPER_LIBRARY})
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory (tests)
|
||||
|
||||
@ -102,4 +102,19 @@ private:
|
||||
|
||||
};
|
||||
|
||||
|
||||
class KeeperMultiException : public KeeperException
|
||||
{
|
||||
public:
|
||||
MultiTransactionInfo info;
|
||||
|
||||
/// If it is user error throws KeeperMultiException else throws ordinary KeeperException
|
||||
/// If it is ZOK does nothing
|
||||
static void check(const MultiTransactionInfo & info);
|
||||
static void check(int code, const Ops & ops, const OpResultsPtr & op_results);
|
||||
|
||||
protected:
|
||||
KeeperMultiException(const MultiTransactionInfo & info, size_t failed_op_index);
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
@ -12,6 +12,8 @@ namespace zkutil
|
||||
|
||||
using ACLPtr = const ACL_vector *;
|
||||
using Stat = ::Stat;
|
||||
class ZooKeeper;
|
||||
|
||||
|
||||
struct Op
|
||||
{
|
||||
@ -19,7 +21,7 @@ public:
|
||||
Op() : data(new zoo_op_t) {}
|
||||
virtual ~Op() {}
|
||||
|
||||
virtual std::unique_ptr<Op> clone() const = 0;
|
||||
virtual std::shared_ptr<Op> clone() const = 0;
|
||||
|
||||
virtual std::string getPath() const = 0;
|
||||
|
||||
@ -33,6 +35,9 @@ public:
|
||||
struct Check;
|
||||
};
|
||||
|
||||
using OpPtr = std::shared_ptr<Op>;
|
||||
|
||||
|
||||
struct Op::Remove : public Op
|
||||
{
|
||||
Remove(const std::string & path_, int32_t version_) :
|
||||
@ -41,9 +46,9 @@ struct Op::Remove : public Op
|
||||
zoo_delete_op_init(data.get(), path.c_str(), version);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
OpPtr clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Remove(path, version));
|
||||
return std::make_shared<Remove>(path, version);
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
@ -59,9 +64,9 @@ struct Op::Create : public Op
|
||||
{
|
||||
Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_);
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
OpPtr clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Create(path_pattern, value, acl, flags));
|
||||
return std::make_shared<Create>(path_pattern, value, acl, flags);
|
||||
}
|
||||
|
||||
std::string getPathCreated() { return created_path.data(); }
|
||||
@ -91,9 +96,9 @@ struct Op::SetData : public Op
|
||||
zoo_set_op_init(data.get(), path.c_str(), value.c_str(), value.size(), version, &stat);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
OpPtr clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new SetData(path, value, version));
|
||||
return std::make_shared<SetData>(path, value, version);
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
@ -122,9 +127,9 @@ struct Op::Check : public Op
|
||||
zoo_check_op_init(data.get(), path.c_str(), version);
|
||||
}
|
||||
|
||||
std::unique_ptr<Op> clone() const override
|
||||
OpPtr clone() const override
|
||||
{
|
||||
return std::unique_ptr<zkutil::Op>(new Check(path, version));
|
||||
return std::make_shared<Check>(path, version);
|
||||
}
|
||||
|
||||
std::string getPath() const override { return path; }
|
||||
@ -136,18 +141,46 @@ private:
|
||||
int32_t version;
|
||||
};
|
||||
|
||||
struct OpResult : public zoo_op_result_t
|
||||
{
|
||||
/// Pointers in this class point to fields of class Op.
|
||||
/// Op instances have the same (or longer lifetime), therefore destructor is not required.
|
||||
};
|
||||
|
||||
using OpPtr = std::unique_ptr<Op>;
|
||||
using Ops = std::vector<OpPtr>;
|
||||
|
||||
|
||||
/// C++ version of zoo_op_result_t
|
||||
struct OpResult
|
||||
{
|
||||
int err;
|
||||
std::string value;
|
||||
std::unique_ptr<Stat> stat;
|
||||
|
||||
/// ZooKeeper is required for correct chroot path prefixes handling
|
||||
explicit OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper = nullptr);
|
||||
};
|
||||
using OpResults = std::vector<OpResult>;
|
||||
using OpResultsPtr = std::shared_ptr<OpResults>;
|
||||
using Strings = std::vector<std::string>;
|
||||
|
||||
|
||||
/// Simple structure to handle transaction execution results
|
||||
struct MultiTransactionInfo
|
||||
{
|
||||
Ops ops;
|
||||
int32_t code = ZOK;
|
||||
OpResultsPtr op_results;
|
||||
|
||||
MultiTransactionInfo() = default;
|
||||
|
||||
MultiTransactionInfo(int32_t code_, const Ops & ops_, const OpResultsPtr & op_results_)
|
||||
: ops(ops_), code(code_), op_results(op_results_) {}
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return ops.empty();
|
||||
}
|
||||
|
||||
/// Returns failed op if zkutil::isUserError(code) is true
|
||||
const Op & getFailedOp() const;
|
||||
};
|
||||
|
||||
|
||||
namespace CreateMode
|
||||
{
|
||||
extern const int Persistent;
|
||||
|
||||
@ -82,19 +82,21 @@ void ZooKeeper::processCallback(zhandle_t *, int type, int state, const char * p
|
||||
}
|
||||
|
||||
void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
int32_t session_timeout_ms_, bool check_root_exists)
|
||||
int32_t session_timeout_ms_, const std::string & chroot_)
|
||||
{
|
||||
log = &Logger::get("ZooKeeper");
|
||||
zoo_set_debug_level(ZOO_LOG_LEVEL_ERROR);
|
||||
hosts = hosts_;
|
||||
identity = identity_;
|
||||
session_timeout_ms = session_timeout_ms_;
|
||||
chroot = chroot_;
|
||||
|
||||
impl = zookeeper_init(hosts.c_str(), nullptr, session_timeout_ms, nullptr, nullptr, 0);
|
||||
std::string hosts_for_lib = hosts + chroot;
|
||||
impl = zookeeper_init(hosts_for_lib.c_str(), nullptr, session_timeout_ms, nullptr, nullptr, 0);
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperInit);
|
||||
|
||||
if (!impl)
|
||||
throw KeeperException("Fail to initialize zookeeper. Hosts are " + hosts);
|
||||
throw KeeperException("Fail to initialize zookeeper. Hosts are " + hosts_for_lib);
|
||||
|
||||
if (!identity.empty())
|
||||
{
|
||||
@ -107,16 +109,16 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
else
|
||||
default_acl = &ZOO_OPEN_ACL_UNSAFE;
|
||||
|
||||
LOG_TRACE(log, "initialized, hosts: " << hosts);
|
||||
LOG_TRACE(log, "initialized, hosts: " << hosts << (chroot.empty() ? "" : ", chroot: " + chroot));
|
||||
|
||||
if (check_root_exists && !exists("/"))
|
||||
throw KeeperException("Zookeeper root doesn't exist. You should create root node before start.");
|
||||
if (!chroot.empty() && !exists("/"))
|
||||
throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.");
|
||||
}
|
||||
|
||||
ZooKeeper::ZooKeeper(const std::string & hosts, const std::string & identity,
|
||||
int32_t session_timeout_ms, bool check_root_exists)
|
||||
int32_t session_timeout_ms, const std::string & chroot)
|
||||
{
|
||||
init(hosts, identity, session_timeout_ms, check_root_exists);
|
||||
init(hosts, identity, session_timeout_ms, chroot);
|
||||
}
|
||||
|
||||
struct ZooKeeperArgs
|
||||
@ -127,10 +129,8 @@ struct ZooKeeperArgs
|
||||
config.keys(config_name, keys);
|
||||
|
||||
std::vector<std::string> hosts_strings;
|
||||
std::string root;
|
||||
|
||||
session_timeout_ms = DEFAULT_SESSION_TIMEOUT;
|
||||
has_chroot = false;
|
||||
for (const auto & key : keys)
|
||||
{
|
||||
if (startsWith(key, "node"))
|
||||
@ -150,7 +150,7 @@ struct ZooKeeperArgs
|
||||
}
|
||||
else if (key == "root")
|
||||
{
|
||||
root = config.getString(config_name + "." + key);
|
||||
chroot = config.getString(config_name + "." + key);
|
||||
}
|
||||
else throw KeeperException(std::string("Unknown key ") + key + " in config file");
|
||||
}
|
||||
@ -166,28 +166,25 @@ struct ZooKeeperArgs
|
||||
hosts += host;
|
||||
}
|
||||
|
||||
if (!root.empty())
|
||||
if (!chroot.empty())
|
||||
{
|
||||
if (root.front() != '/')
|
||||
throw KeeperException(std::string("Root path in config file should start with '/', but got ") + root);
|
||||
if (root.back() == '/')
|
||||
root.pop_back();
|
||||
|
||||
hosts += root;
|
||||
has_chroot = true;
|
||||
if (chroot.front() != '/')
|
||||
throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot);
|
||||
if (chroot.back() == '/')
|
||||
chroot.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
std::string hosts;
|
||||
std::string identity;
|
||||
int session_timeout_ms;
|
||||
bool has_chroot;
|
||||
std::string chroot;
|
||||
};
|
||||
|
||||
ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name)
|
||||
{
|
||||
ZooKeeperArgs args(config, config_name);
|
||||
init(args.hosts, args.identity, args.session_timeout_ms, args.has_chroot);
|
||||
init(args.hosts, args.identity, args.session_timeout_ms, args.chroot);
|
||||
}
|
||||
|
||||
WatchCallback ZooKeeper::callbackForEvent(const EventPtr & event)
|
||||
@ -290,19 +287,18 @@ int32_t ZooKeeper::createImpl(const std::string & path, const std::string & data
|
||||
int code;
|
||||
/// The name of the created node can be longer than path if the sequential node is created.
|
||||
size_t name_buffer_size = path.size() + SEQUENTIAL_SUFFIX_SIZE;
|
||||
char * name_buffer = new char[name_buffer_size];
|
||||
std::string name_buffer(name_buffer_size, '\0');
|
||||
|
||||
code = zoo_create(impl, path.c_str(), data.c_str(), data.size(), getDefaultACL(), mode, name_buffer, name_buffer_size);
|
||||
code = zoo_create(impl, path.c_str(), data.c_str(), data.size(), getDefaultACL(), mode, name_buffer.data(), name_buffer_size);
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperCreate);
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions);
|
||||
|
||||
if (code == ZOK)
|
||||
{
|
||||
path_created = std::string(name_buffer);
|
||||
name_buffer.resize(strlen(name_buffer.data()));
|
||||
path_created = std::move(name_buffer);
|
||||
}
|
||||
|
||||
delete[] name_buffer;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
@ -571,7 +567,19 @@ int32_t ZooKeeper::trySet(const std::string & path, const std::string & data,
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
|
||||
/// Makes deep copy of zoo_op_result_t and removes chroot prefix from paths
|
||||
static void convertOpResults(const std::vector<zoo_op_result_t> & op_results_native, OpResultsPtr & out_op_results,
|
||||
const ZooKeeper * zookeeper = nullptr)
|
||||
{
|
||||
if (!out_op_results)
|
||||
out_op_results = std::make_shared<OpResults>();
|
||||
|
||||
out_op_results->reserve(op_results_native.size());
|
||||
for (const zoo_op_result_t & res_native : op_results_native)
|
||||
out_op_results->emplace_back(res_native, zookeeper);
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_op_results, MultiTransactionInfo * out_info)
|
||||
{
|
||||
if (ops_.empty())
|
||||
return ZOK;
|
||||
@ -585,7 +593,7 @@ int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
|
||||
return ZINVALIDSTATE;
|
||||
|
||||
size_t count = ops_.size();
|
||||
OpResultsPtr out_results(new OpResults(count));
|
||||
std::vector<zoo_op_result_t> out_results_native(count);
|
||||
|
||||
/// Copy the struct containing pointers with default copy-constructor.
|
||||
/// It is safe because it hasn't got a destructor.
|
||||
@ -594,34 +602,31 @@ int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
|
||||
for (const auto & op : ops_)
|
||||
ops.push_back(*(op->data));
|
||||
|
||||
int32_t code = zoo_multi(impl, static_cast<int>(ops.size()), ops.data(), out_results->data());
|
||||
int32_t code = zoo_multi(impl, static_cast<int>(ops.size()), ops.data(), out_results_native.data());
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperMulti);
|
||||
ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions);
|
||||
|
||||
if (out_results_)
|
||||
*out_results_ = out_results;
|
||||
if (out_op_results || out_info)
|
||||
{
|
||||
OpResultsPtr op_results;
|
||||
convertOpResults(out_results_native, op_results, this);
|
||||
|
||||
if (out_op_results)
|
||||
*out_op_results = op_results;
|
||||
|
||||
if (out_info)
|
||||
*out_info = MultiTransactionInfo(code, ops_, op_results);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
OpResultsPtr ZooKeeper::multi(const Ops & ops)
|
||||
{
|
||||
OpResultsPtr results;
|
||||
int code = tryMulti(ops, &results);
|
||||
if (code != ZOK)
|
||||
{
|
||||
if (results && results->size() == ops.size())
|
||||
{
|
||||
for (size_t i = 0; i < ops.size(); ++i)
|
||||
{
|
||||
if (results->at(i).err == code)
|
||||
throw KeeperException("Transaction failed at op #" + std::to_string(i) + ": " + ops[i]->describe(), code);
|
||||
}
|
||||
}
|
||||
|
||||
throw KeeperException(code);
|
||||
}
|
||||
return results;
|
||||
OpResultsPtr op_results;
|
||||
int code = multiImpl(ops, &op_results);
|
||||
KeeperMultiException::check(code, ops, op_results);
|
||||
return op_results;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryMulti(const Ops & ops_, OpResultsPtr * out_results_)
|
||||
@ -638,17 +643,9 @@ int32_t ZooKeeper::tryMulti(const Ops & ops_, OpResultsPtr * out_results_)
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryMultiUnsafe(const Ops & ops, MultiTransactionInfo & info)
|
||||
{
|
||||
info.code = multiImpl(ops, &info.op_results);
|
||||
for (const OpPtr & op : ops)
|
||||
info.ops.emplace_back(op->clone());
|
||||
return info.code;
|
||||
}
|
||||
|
||||
int32_t ZooKeeper::tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results, size_t * attempt)
|
||||
{
|
||||
int32_t code = retry(std::bind(&ZooKeeper::multiImpl, this, std::ref(ops), out_results), attempt);
|
||||
int32_t code = retry(std::bind(&ZooKeeper::multiImpl, this, std::ref(ops), out_results, nullptr), attempt);
|
||||
if (!(code == ZOK ||
|
||||
code == ZNONODE ||
|
||||
code == ZNODEEXISTS ||
|
||||
@ -754,7 +751,7 @@ ZooKeeper::~ZooKeeper()
|
||||
|
||||
ZooKeeperPtr ZooKeeper::startNewSession() const
|
||||
{
|
||||
return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms);
|
||||
return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms, chroot);
|
||||
}
|
||||
|
||||
Op::Create::Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_)
|
||||
@ -981,26 +978,27 @@ ZooKeeper::TryRemoveFuture ZooKeeper::asyncTryRemove(const std::string & path, i
|
||||
|
||||
ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception)
|
||||
{
|
||||
size_t count = ops_.size();
|
||||
OpResultsPtr results(new OpResults(count));
|
||||
|
||||
/// We need to hold all references to ops data until the end of multi callback
|
||||
struct OpsHolder
|
||||
{
|
||||
std::shared_ptr<zkutil::Ops> ops_ptr = std::make_shared<zkutil::Ops>();
|
||||
std::shared_ptr<std::vector<zoo_op_t>> ops_raw_ptr = std::make_shared<std::vector<zoo_op_t>>();
|
||||
std::shared_ptr<zkutil::Ops> ops_ptr;
|
||||
std::shared_ptr<std::vector<zoo_op_t>> ops_native;
|
||||
std::shared_ptr<std::vector<zoo_op_result_t>> op_results_native;
|
||||
} holder;
|
||||
|
||||
for (const auto & op : ops_)
|
||||
{
|
||||
holder.ops_ptr->emplace_back(op->clone());
|
||||
holder.ops_raw_ptr->push_back(*holder.ops_ptr->back()->data);
|
||||
}
|
||||
/// Copy ops (swallow copy)
|
||||
holder.ops_ptr = std::make_shared<zkutil::Ops>(ops_);
|
||||
/// Copy native ops to contiguous vector
|
||||
holder.ops_native = std::make_shared<std::vector<zoo_op_t>>();
|
||||
for (const OpPtr & op : *holder.ops_ptr)
|
||||
holder.ops_native->push_back(*op->data);
|
||||
/// Allocate native result holders
|
||||
holder.op_results_native = std::make_shared<std::vector<zoo_op_result_t>>(holder.ops_ptr->size());
|
||||
|
||||
MultiFuture future{ [throw_exception, results, holder] (int rc) {
|
||||
MultiFuture future{ [throw_exception, holder, zookeeper=this] (int rc) {
|
||||
OpResultsAndCode res;
|
||||
res.code = rc;
|
||||
res.results = results;
|
||||
convertOpResults(*holder.op_results_native, res.results, zookeeper);
|
||||
res.ops_ptr = holder.ops_ptr;
|
||||
if (throw_exception && rc != ZOK)
|
||||
throw zkutil::KeeperException(rc);
|
||||
@ -1018,9 +1016,9 @@ ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool
|
||||
if (expired())
|
||||
throw KeeperException(ZINVALIDSTATE);
|
||||
|
||||
auto & ops = *holder.ops_raw_ptr;
|
||||
|
||||
int32_t code = zoo_amulti(impl, static_cast<int>(ops.size()), ops.data(), results->data(),
|
||||
int32_t code = zoo_amulti(impl, static_cast<int>(holder.ops_native->size()),
|
||||
holder.ops_native->data(),
|
||||
holder.op_results_native->data(),
|
||||
[] (int rc, const void * data)
|
||||
{
|
||||
MultiFuture::TaskPtr owned_task =
|
||||
@ -1069,4 +1067,58 @@ size_t getFailedOpIndex(const OpResultsPtr & op_results, int32_t transaction_ret
|
||||
}
|
||||
|
||||
|
||||
OpResult::OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper)
|
||||
: err(op_result.err)
|
||||
{
|
||||
if (op_result.value)
|
||||
{
|
||||
value = std::string(op_result.value, op_result.value + op_result.valuelen);
|
||||
|
||||
/// Current version of libzookeeper does not cut chroot path prefixes
|
||||
/// We do it here manually
|
||||
if (zookeeper && !zookeeper->chroot.empty())
|
||||
{
|
||||
if (startsWith(value, zookeeper->chroot))
|
||||
value = value.substr(zookeeper->chroot.length());
|
||||
else
|
||||
throw DB::Exception("Expected ZooKeeper path with chroot " + zookeeper->chroot + ", got " + value,
|
||||
DB::ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
if (op_result.stat)
|
||||
stat = std::make_unique<Stat>(*op_result.stat);
|
||||
}
|
||||
|
||||
|
||||
KeeperMultiException::KeeperMultiException(const MultiTransactionInfo & info_, size_t failed_op_index_)
|
||||
:KeeperException(
|
||||
"Transaction failed at op #" + std::to_string(failed_op_index_) + ": " + info_.ops.at(failed_op_index_)->describe(),
|
||||
info_.code),
|
||||
info(info_) {}
|
||||
|
||||
void KeeperMultiException::check(int code, const Ops & ops, const OpResultsPtr & op_results)
|
||||
{
|
||||
if (code == ZOK) {}
|
||||
else if (zkutil::isUserError(code))
|
||||
throw KeeperMultiException(MultiTransactionInfo(code, ops, op_results), getFailedOpIndex(op_results, code));
|
||||
else
|
||||
throw KeeperException(code);
|
||||
}
|
||||
|
||||
void KeeperMultiException::check(const MultiTransactionInfo & info)
|
||||
{
|
||||
if (info.code == ZOK) {}
|
||||
else if (zkutil::isUserError(info.code))
|
||||
throw KeeperMultiException(info, getFailedOpIndex(info.op_results, info.code));
|
||||
else
|
||||
throw KeeperException(info.code);
|
||||
}
|
||||
|
||||
|
||||
const Op & MultiTransactionInfo::getFailedOp() const
|
||||
{
|
||||
return *ops.at(getFailedOpIndex(op_results, code));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -8,10 +8,10 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unistd.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
@ -56,7 +56,7 @@ public:
|
||||
using Ptr = std::shared_ptr<ZooKeeper>;
|
||||
|
||||
ZooKeeper(const std::string & hosts, const std::string & identity = "",
|
||||
int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, bool check_root_exists = false);
|
||||
int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, const std::string & chroot = "");
|
||||
|
||||
/** Config of the form:
|
||||
<zookeeper>
|
||||
@ -196,14 +196,16 @@ public:
|
||||
/// Performs several operations in a transaction.
|
||||
/// Throws on every error.
|
||||
OpResultsPtr multi(const Ops & ops);
|
||||
|
||||
/// Throws only if some operation has returned an "unexpected" error
|
||||
/// - an error that would cause the corresponding try- method to throw.
|
||||
int32_t tryMulti(const Ops & ops, OpResultsPtr * out_results = nullptr);
|
||||
/// Like previous one, but does not throw any ZooKeeper exceptions
|
||||
int32_t tryMultiUnsafe(const Ops & ops, MultiTransactionInfo & info);
|
||||
/// Use only with read-only operations.
|
||||
int32_t tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results = nullptr, size_t * attempt = nullptr);
|
||||
/// Throws nothing, just alias of multiImpl
|
||||
int32_t tryMultiNoThrow(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr)
|
||||
{
|
||||
return multiImpl(ops, out_op_results, out_info);
|
||||
}
|
||||
|
||||
Int64 getClientID();
|
||||
|
||||
@ -366,9 +368,10 @@ public:
|
||||
private:
|
||||
friend struct WatchContext;
|
||||
friend class EphemeralNodeHolder;
|
||||
friend struct OpResult;
|
||||
|
||||
void init(const std::string & hosts, const std::string & identity, int32_t session_timeout_ms, const std::string & chroot);
|
||||
|
||||
void init(const std::string & hosts, const std::string & identity,
|
||||
int32_t session_timeout_ms, bool check_root_exists);
|
||||
void removeChildrenRecursive(const std::string & path);
|
||||
void tryRemoveChildrenRecursive(const std::string & path);
|
||||
|
||||
@ -391,7 +394,7 @@ private:
|
||||
/// If the connection has been lost, wait timeout/3 hoping for connection re-establishment.
|
||||
static const int MAX_SLEEP_TIME = 10;
|
||||
if (code == ZCONNECTIONLOSS)
|
||||
usleep(std::min(session_timeout_ms * 1000 / 3, MAX_SLEEP_TIME * 1000 * 1000));
|
||||
usleep(std::min(session_timeout_ms * 1000u / 3, MAX_SLEEP_TIME * 1000u * 1000u));
|
||||
|
||||
LOG_WARNING(log, "Error on attempt " << i << ": " << error2string(code) << ". Retry");
|
||||
code = operation();
|
||||
@ -406,7 +409,7 @@ private:
|
||||
int32_t getImpl(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback);
|
||||
int32_t setImpl(const std::string & path, const std::string & data, int32_t version = -1, Stat * stat = nullptr);
|
||||
int32_t getChildrenImpl(const std::string & path, Strings & res, Stat * stat, WatchCallback watch_callback);
|
||||
int32_t multiImpl(const Ops & ops, OpResultsPtr * out_results = nullptr);
|
||||
int32_t multiImpl(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr);
|
||||
int32_t existsImpl(const std::string & path, Stat * stat_, WatchCallback watch_callback);
|
||||
|
||||
MultiFuture asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception);
|
||||
@ -414,6 +417,7 @@ private:
|
||||
std::string hosts;
|
||||
std::string identity;
|
||||
int32_t session_timeout_ms;
|
||||
std::string chroot;
|
||||
|
||||
std::mutex mutex;
|
||||
ACLPtr default_acl;
|
||||
@ -493,41 +497,4 @@ private:
|
||||
|
||||
using EphemeralNodeHolderPtr = EphemeralNodeHolder::Ptr;
|
||||
|
||||
|
||||
/// Simple structure to handle transaction execution results
|
||||
struct MultiTransactionInfo
|
||||
{
|
||||
MultiTransactionInfo() = default;
|
||||
|
||||
Ops ops;
|
||||
int32_t code = ZOK;
|
||||
OpResultsPtr op_results;
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
return ops.empty();
|
||||
}
|
||||
|
||||
bool hasFailedOp() const
|
||||
{
|
||||
return zkutil::isUserError(code);
|
||||
}
|
||||
|
||||
const Op & getFailedOp() const
|
||||
{
|
||||
return *ops.at(getFailedOpIndex(op_results, code));
|
||||
}
|
||||
|
||||
KeeperException getException() const
|
||||
{
|
||||
if (hasFailedOp())
|
||||
{
|
||||
size_t i = getFailedOpIndex(op_results, code);
|
||||
return KeeperException("Transaction failed at op #" + std::to_string(i) + ": " + ops.at(i)->describe(), code);
|
||||
}
|
||||
else
|
||||
return KeeperException(code);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -6,6 +6,9 @@
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
#include <gtest/gtest.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
using namespace DB;
|
||||
@ -32,32 +35,32 @@ TEST(zkutil, multi_nice_exception_msg)
|
||||
zkutil::Ops ops;
|
||||
|
||||
ASSERT_NO_THROW(
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
|
||||
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
zookeeper->multi(ops);
|
||||
);
|
||||
|
||||
try
|
||||
{
|
||||
ops.clear();
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/c", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test_zkutil_multi/c", -1));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "BadBoy", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/b", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/c", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi/c", -1));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "BadBoy", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/b", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
|
||||
zookeeper->multi(ops);
|
||||
FAIL();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
|
||||
|
||||
String msg = getCurrentExceptionMessage(false);
|
||||
|
||||
bool msg_has_reqired_patterns = msg.find("/clickhouse_test_zkutil_multi/a") != std::string::npos && msg.find("#2") != std::string::npos;
|
||||
bool msg_has_reqired_patterns = msg.find("/clickhouse_test/zkutil_multi/a") != std::string::npos && msg.find("#2") != std::string::npos;
|
||||
EXPECT_TRUE(msg_has_reqired_patterns) << msg;
|
||||
}
|
||||
}
|
||||
@ -69,7 +72,7 @@ TEST(zkutil, multi_async)
|
||||
auto acl = zookeeper->getDefaultACL();
|
||||
zkutil::Ops ops;
|
||||
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
|
||||
zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
|
||||
|
||||
{
|
||||
ops.clear();
|
||||
@ -78,8 +81,8 @@ TEST(zkutil, multi_async)
|
||||
|
||||
{
|
||||
ops.clear();
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "", acl, zkutil::CreateMode::Persistent));
|
||||
|
||||
auto fut = zookeeper->tryAsyncMulti(ops);
|
||||
ops.clear();
|
||||
@ -97,11 +100,11 @@ TEST(zkutil, multi_async)
|
||||
for (size_t i = 0; i < 10000; ++i)
|
||||
{
|
||||
ops.clear();
|
||||
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test_zkutil_multi", -1));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Check("/clickhouse_test_zkutil_multi", -1));
|
||||
ops.emplace_back(new zkutil::Op::SetData("/clickhouse_test_zkutil_multi", "xxx", 42));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi", -1));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", -1));
|
||||
ops.emplace_back(new zkutil::Op::SetData("/clickhouse_test/zkutil_multi", "xxx", 42));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
|
||||
futures.emplace_back(zookeeper->asyncMulti(ops));
|
||||
}
|
||||
@ -115,8 +118,8 @@ TEST(zkutil, multi_async)
|
||||
|
||||
{
|
||||
ops.clear();
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
|
||||
|
||||
auto fut = zookeeper->tryAsyncMulti(ops);
|
||||
ops.clear();
|
||||
@ -128,31 +131,55 @@ TEST(zkutil, multi_async)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run this test under sudo
|
||||
TEST(zkutil, multi_async_libzookeeper_segfault)
|
||||
{
|
||||
auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", 1000);
|
||||
zkutil::Ops ops;
|
||||
|
||||
ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", 0));
|
||||
|
||||
/// Uncomment to test
|
||||
//auto cmd = ShellCommand::execute("sudo service zookeeper restart");
|
||||
//cmd->wait();
|
||||
|
||||
auto future = zookeeper->asyncMulti(ops);
|
||||
auto res = future.get();
|
||||
|
||||
EXPECT_TRUE(zkutil::isUnrecoverableErrorCode(res.code));
|
||||
}
|
||||
|
||||
|
||||
TEST(zkutil, multi_create_sequential)
|
||||
{
|
||||
try
|
||||
{
|
||||
/// Create chroot node firstly
|
||||
auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
|
||||
zookeeper->createAncestors("/clickhouse_test/");
|
||||
|
||||
zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", zkutil::DEFAULT_SESSION_TIMEOUT, "/clickhouse_test");
|
||||
auto acl = zookeeper->getDefaultACL();
|
||||
zkutil::Ops ops;
|
||||
|
||||
String base_path = "/clickhouse_test/zkutil/multi_create_sequential";
|
||||
String base_path = "/zkutil/multi_create_sequential";
|
||||
zookeeper->tryRemoveRecursive(base_path);
|
||||
zookeeper->createAncestors(base_path + "/");
|
||||
|
||||
String entry_path = base_path + "/queue-";
|
||||
ops.emplace_back(new zkutil::Op::Create(entry_path, "", acl, zkutil::CreateMode::EphemeralSequential));
|
||||
String sequential_node_prefix = base_path + "/queue-";
|
||||
ops.emplace_back(new zkutil::Op::Create(sequential_node_prefix, "", acl, zkutil::CreateMode::EphemeralSequential));
|
||||
zkutil::OpResultsPtr results = zookeeper->multi(ops);
|
||||
zkutil::OpResult & result = results->at(0);
|
||||
zkutil::OpResult & sequential_node_result_op = results->at(0);
|
||||
|
||||
EXPECT_TRUE(result.value != nullptr);
|
||||
EXPECT_TRUE(startsWith(result.value, entry_path));
|
||||
EXPECT_FALSE(sequential_node_result_op.value.empty());
|
||||
EXPECT_GT(sequential_node_result_op.value.length(), sequential_node_prefix.length());
|
||||
EXPECT_EQ(sequential_node_result_op.value.substr(0, sequential_node_prefix.length()), sequential_node_prefix);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << getCurrentExceptionMessage(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -34,11 +34,11 @@ int main(int argc, char ** argv)
|
||||
{
|
||||
{
|
||||
zkutil::Ops ops;
|
||||
ops.emplace_back(std::make_unique<zkutil::Op::Create>("/test/zk_expiration_test", "hello", zk.getDefaultACL(), zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(std::make_unique<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
|
||||
ops.emplace_back(std::make_shared<zkutil::Op::Create>("/test/zk_expiration_test", "hello", zk.getDefaultACL(), zkutil::CreateMode::Persistent));
|
||||
ops.emplace_back(std::make_shared<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
|
||||
|
||||
zkutil::MultiTransactionInfo info;
|
||||
zk.tryMultiUnsafe(ops, info);
|
||||
zk.tryMultiNoThrow(ops, nullptr, &info);
|
||||
|
||||
std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(info.code) << std::endl;
|
||||
try
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <iostream>
|
||||
#include <unistd.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
using namespace zkutil;
|
||||
|
||||
|
||||
@ -12,7 +12,6 @@ namespace Poco
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Lets you check if the address is similar to `localhost`.
|
||||
* The purpose of this check is usually to make an assumption,
|
||||
* that when we go to this address via the Internet, we'll get to ourselves.
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Poco/File.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <port/unistd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -1,12 +1,10 @@
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <port/unistd.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Core/Types.h>
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <common/apple_rt.h>
|
||||
#endif
|
||||
|
||||
@ -7,15 +7,13 @@
|
||||
#include <Common/CompactArray.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <unistd.h>
|
||||
#include <cstdlib>
|
||||
#include <port/unistd.h>
|
||||
|
||||
namespace fs = boost::filesystem;
|
||||
|
||||
|
||||
@ -312,7 +312,7 @@ MutableColumns Block::mutateColumns() const
|
||||
size_t num_columns = data.size();
|
||||
MutableColumns columns(num_columns);
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
columns[i] = data[i].column ? data[i].column->mutate() : data[i].type->createColumn();
|
||||
columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn();
|
||||
return columns;
|
||||
}
|
||||
|
||||
|
||||
@ -109,7 +109,7 @@ public:
|
||||
/** Get empty columns with the same types as in block. */
|
||||
MutableColumns cloneEmptyColumns() const;
|
||||
|
||||
/** Get columns from block for mutation. */
|
||||
/** Get columns from block for mutation. Columns in block will be nullptr. */
|
||||
MutableColumns mutateColumns() const;
|
||||
|
||||
/** Replace columns in a block */
|
||||
|
||||
@ -60,6 +60,7 @@
|
||||
#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
|
||||
#define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
|
||||
#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
|
||||
#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
|
||||
|
||||
/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
|
||||
#define DBMS_TCP_PROTOCOL_VERSION 54226
|
||||
@ -74,8 +75,18 @@
|
||||
#define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
|
||||
#define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
|
||||
|
||||
#define ALWAYS_INLINE __attribute__((__always_inline__))
|
||||
#define NO_INLINE __attribute__((__noinline__))
|
||||
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define ALWAYS_INLINE __forceinline
|
||||
#define NO_INLINE static __declspec(noinline)
|
||||
#define MAY_ALIAS
|
||||
#else
|
||||
#define ALWAYS_INLINE __attribute__((__always_inline__))
|
||||
#define NO_INLINE __attribute__((__noinline__))
|
||||
#define MAY_ALIAS __attribute__((__may_alias__))
|
||||
#endif
|
||||
|
||||
|
||||
#define PLATFORM_NOT_SUPPORTED "The only supported platforms are x86_64 and AArch64 (work in progress)"
|
||||
|
||||
|
||||
@ -198,14 +198,14 @@ public:
|
||||
template <typename T> T & get()
|
||||
{
|
||||
using TWithoutRef = std::remove_reference_t<T>;
|
||||
TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<TWithoutRef*>(&storage);
|
||||
TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<TWithoutRef*>(&storage);
|
||||
return *ptr;
|
||||
};
|
||||
|
||||
template <typename T> const T & get() const
|
||||
{
|
||||
using TWithoutRef = std::remove_reference_t<T>;
|
||||
const TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<const TWithoutRef*>(&storage);
|
||||
const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<const TWithoutRef*>(&storage);
|
||||
return *ptr;
|
||||
};
|
||||
|
||||
@ -340,7 +340,7 @@ private:
|
||||
void createConcrete(T && x)
|
||||
{
|
||||
using JustT = std::decay_t<T>;
|
||||
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
|
||||
JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
|
||||
new (ptr) JustT(std::forward<T>(x));
|
||||
which = TypeToEnum<JustT>::value;
|
||||
}
|
||||
@ -350,7 +350,7 @@ private:
|
||||
void assignConcrete(T && x)
|
||||
{
|
||||
using JustT = std::decay_t<T>;
|
||||
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
|
||||
JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
|
||||
*ptr = std::forward<T>(x);
|
||||
}
|
||||
|
||||
@ -398,7 +398,7 @@ private:
|
||||
|
||||
void create(const char * data, size_t size)
|
||||
{
|
||||
String * __attribute__((__may_alias__)) ptr = reinterpret_cast<String*>(&storage);
|
||||
String * MAY_ALIAS ptr = reinterpret_cast<String*>(&storage);
|
||||
new (ptr) String(data, size);
|
||||
which = Types::String;
|
||||
}
|
||||
@ -434,7 +434,7 @@ private:
|
||||
template <typename T>
|
||||
void destroy()
|
||||
{
|
||||
T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(&storage);
|
||||
T * MAY_ALIAS ptr = reinterpret_cast<T*>(&storage);
|
||||
ptr->~T();
|
||||
}
|
||||
};
|
||||
|
||||
@ -138,4 +138,14 @@ NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const
|
||||
return res;
|
||||
}
|
||||
|
||||
bool NamesAndTypesList::contains(const String & name) const
|
||||
{
|
||||
for (const NameAndTypePair & column : *this)
|
||||
{
|
||||
if (column.name == name)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -69,6 +69,8 @@ public:
|
||||
|
||||
/// Unlike `filter`, returns columns in the order in which they go in `names`.
|
||||
NamesAndTypesList addTypes(const Names & names) const;
|
||||
|
||||
bool contains(const String & name) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -116,7 +116,7 @@ namespace Protocol
|
||||
};
|
||||
|
||||
/// Whether the ssl must be used.
|
||||
enum class Encryption
|
||||
enum class Secure
|
||||
{
|
||||
Disable = 0,
|
||||
Enable = 1,
|
||||
|
||||
@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
|
||||
std::ostream & operator<<(std::ostream & stream, const IStorage & what)
|
||||
{
|
||||
stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
|
||||
<< what.getColumnsList().toString()
|
||||
<< what.getColumns().getAllPhysical().toString()
|
||||
<< "}";
|
||||
// isRemote supportsSampling supportsFinal supportsPrewhere
|
||||
return stream;
|
||||
|
||||
@ -76,7 +76,7 @@ Block ColumnGathererStream::readImpl()
|
||||
return Block();
|
||||
|
||||
output_block = Block{column.cloneEmpty()};
|
||||
MutableColumnPtr output_column = output_block.getByPosition(0).column->mutate();
|
||||
MutableColumnPtr output_column = output_block.getByPosition(0).column->assumeMutable();
|
||||
output_column->gather(*this);
|
||||
if (!output_column->empty())
|
||||
output_block.getByPosition(0).column = std::move(output_column);
|
||||
|
||||
@ -221,7 +221,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
|
||||
throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
merged_columns[i] = source_blocks[source_num]->getByPosition(i).column->mutate();
|
||||
merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate();
|
||||
|
||||
// std::cerr << "copied columns\n";
|
||||
|
||||
@ -233,7 +233,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
{
|
||||
auto & column = merged_columns[i];
|
||||
column = column->cut(0, merged_rows);
|
||||
column = (*column->cut(0, merged_rows)).mutate();
|
||||
}
|
||||
|
||||
cancel(false);
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#include <sys/ioctl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <port/unistd.h>
|
||||
#include <DataStreams/PrettyBlockOutputStream.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <DataStreams/PushingToViewsBlockOutputStream.h>
|
||||
#include <DataStreams/SquashingBlockInputStream.h>
|
||||
#include <Interpreters/InterpreterSelectQuery.h>
|
||||
#include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
|
||||
|
||||
@ -35,7 +36,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
|
||||
auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);
|
||||
|
||||
auto query = materialized_view.getInnerQuery();
|
||||
auto out = std::make_shared<PushingToViewsBlockOutputStream>(
|
||||
BlockOutputStreamPtr out = std::make_shared<PushingToViewsBlockOutputStream>(
|
||||
database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
|
||||
views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
|
||||
}
|
||||
@ -66,8 +67,19 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
|
||||
{
|
||||
BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
|
||||
InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from);
|
||||
BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
|
||||
copyData(*data, *view.out);
|
||||
BlockInputStreamPtr in = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
|
||||
/// Squashing is needed here because the materialized view query can generate a lot of blocks
|
||||
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
|
||||
/// and two-level aggregation is triggered).
|
||||
in = std::make_shared<SquashingBlockInputStream>(
|
||||
in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
|
||||
|
||||
in->readPrefix();
|
||||
|
||||
while (Block result_block = in->read())
|
||||
view.out->write(result_block);
|
||||
|
||||
in->readSuffix();
|
||||
}
|
||||
catch (Exception & ex)
|
||||
{
|
||||
|
||||
@ -29,18 +29,27 @@ public:
|
||||
{
|
||||
if (output)
|
||||
output->flush();
|
||||
|
||||
for (auto & view : views)
|
||||
view.out->flush();
|
||||
}
|
||||
|
||||
void writePrefix() override
|
||||
{
|
||||
if (output)
|
||||
output->writePrefix();
|
||||
|
||||
for (auto & view : views)
|
||||
view.out->writePrefix();
|
||||
}
|
||||
|
||||
void writeSuffix() override
|
||||
{
|
||||
if (output)
|
||||
output->writeSuffix();
|
||||
|
||||
for (auto & view : views)
|
||||
view.out->writeSuffix();
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables()
|
||||
{
|
||||
StoragePtr cur = table.second;
|
||||
QueryProcessingStage::Enum stage = QueryProcessingStage::Complete;
|
||||
BlockInputStreams input = cur->read(cur->getColumnNamesList(), {}, context,
|
||||
BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context,
|
||||
stage, DEFAULT_BLOCK_SIZE, 1);
|
||||
if (input.size() == 0)
|
||||
res.push_back(std::make_pair(std::make_shared<OneBlockInputStream>(cur->getSampleBlock()), table.first));
|
||||
|
||||
@ -62,7 +62,7 @@ void SquashingTransform::append(Block && block)
|
||||
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
MutableColumnPtr mutable_column = accumulated_block.getByPosition(i).column->mutate();
|
||||
MutableColumnPtr mutable_column = (*std::move(accumulated_block.getByPosition(i).column)).mutate();
|
||||
mutable_column->insertRangeFrom(*block.getByPosition(i).column, 0, rows);
|
||||
accumulated_block.getByPosition(i).column = std::move(mutable_column);
|
||||
}
|
||||
|
||||
@ -259,11 +259,11 @@ Block SummingSortedBlockInputStream::readImpl()
|
||||
if (checkDataType<DataTypeTuple>(desc.function->getReturnType().get()))
|
||||
{
|
||||
size_t tuple_size = desc.column_numbers.size();
|
||||
Columns tuple_columns(tuple_size);
|
||||
MutableColumns tuple_columns(tuple_size);
|
||||
for (size_t i = 0; i < tuple_size; ++i)
|
||||
tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column;
|
||||
tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->assumeMutable();
|
||||
|
||||
desc.merged_column = ColumnTuple::create(tuple_columns);
|
||||
desc.merged_column = ColumnTuple::create(std::move(tuple_columns));
|
||||
}
|
||||
else
|
||||
desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty();
|
||||
|
||||
@ -104,8 +104,8 @@ int main(int, char **)
|
||||
|
||||
/// create an object of an existing hit log table
|
||||
|
||||
StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
|
||||
NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
StoragePtr table = StorageLog::create(
|
||||
"./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
table->startup();
|
||||
|
||||
/// read from it, apply the expression, filter, and write in tsv form to the console
|
||||
|
||||
@ -95,8 +95,8 @@ try
|
||||
|
||||
/// create an object of an existing hit log table
|
||||
|
||||
StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
|
||||
NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
StoragePtr table = StorageLog::create(
|
||||
"./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
table->startup();
|
||||
|
||||
/// read from it
|
||||
|
||||
@ -107,8 +107,8 @@ try
|
||||
|
||||
/// create an object of an existing hit log table
|
||||
|
||||
StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
|
||||
NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
StoragePtr table = StorageLog::create(
|
||||
"./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
|
||||
table->startup();
|
||||
|
||||
/// read from it, sort it, and write it in tsv form to the console
|
||||
|
||||
@ -329,10 +329,10 @@ void DataTypeTuple::deserializeBinaryBulkWithMultipleStreams(
|
||||
MutableColumnPtr DataTypeTuple::createColumn() const
|
||||
{
|
||||
size_t size = elems.size();
|
||||
Columns tuple_columns(size);
|
||||
MutableColumns tuple_columns(size);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
tuple_columns[i] = elems[i]->createColumn();
|
||||
return ColumnTuple::create(tuple_columns);
|
||||
return ColumnTuple::create(std::move(tuple_columns));
|
||||
}
|
||||
|
||||
Field DataTypeTuple::getDefault() const
|
||||
|
||||
@ -3,15 +3,21 @@
|
||||
#include <Interpreters/ExternalDictionaries.h>
|
||||
#include <Storages/StorageDictionary.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TABLE_ALREADY_EXISTS;
|
||||
extern const int UNKNOWN_TABLE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
|
||||
extern const int TABLE_ALREADY_EXISTS;
|
||||
extern const int UNKNOWN_TABLE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
|
||||
extern const int SYNTAX_ERROR;
|
||||
}
|
||||
|
||||
DatabaseDictionary::DatabaseDictionary(const String & name_, const Context & context)
|
||||
@ -41,8 +47,7 @@ Tables DatabaseDictionary::loadTables()
|
||||
{
|
||||
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
|
||||
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
|
||||
tables[name] = StorageDictionary::create(name,
|
||||
columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, name);
|
||||
tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name);
|
||||
}
|
||||
}
|
||||
|
||||
@ -76,8 +81,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
|
||||
{
|
||||
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
|
||||
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
|
||||
return StorageDictionary::create(table_name,
|
||||
columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, table_name);
|
||||
return StorageDictionary::create(table_name, ColumnsDescription{columns}, dictionary_structure, table_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -87,7 +91,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
|
||||
|
||||
DatabaseIteratorPtr DatabaseDictionary::getIterator(const Context & /*context*/)
|
||||
{
|
||||
return std::make_unique<DatabaseSnaphotIterator>(loadTables());
|
||||
return std::make_unique<DatabaseSnapshotIterator>(loadTables());
|
||||
}
|
||||
|
||||
bool DatabaseDictionary::empty(const Context & /*context*/) const
|
||||
@ -142,10 +146,7 @@ void DatabaseDictionary::renameTable(
|
||||
void DatabaseDictionary::alterTable(
|
||||
const Context &,
|
||||
const String &,
|
||||
const NamesAndTypesList &,
|
||||
const NamesAndTypesList &,
|
||||
const NamesAndTypesList &,
|
||||
const ColumnDefaults &,
|
||||
const ColumnsDescription &,
|
||||
const ASTModifier &)
|
||||
{
|
||||
throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
@ -158,11 +159,54 @@ time_t DatabaseDictionary::getTableMetadataModificationTime(
|
||||
return static_cast<time_t>(0);
|
||||
}
|
||||
|
||||
ASTPtr DatabaseDictionary::getCreateQuery(
|
||||
const Context &,
|
||||
const String &) const
|
||||
ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
|
||||
const String & table_name, bool throw_on_error) const
|
||||
{
|
||||
throw Exception("There is no CREATE TABLE query for DatabaseDictionary tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
|
||||
String query;
|
||||
{
|
||||
WriteBufferFromString buffer(query);
|
||||
|
||||
const auto & dictionaries = context.getExternalDictionaries();
|
||||
auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name)
|
||||
: dictionaries.tryGetDictionary(table_name);
|
||||
|
||||
auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
|
||||
buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
|
||||
buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
|
||||
buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
|
||||
}
|
||||
|
||||
ParserCreateQuery parser;
|
||||
const char * pos = query.data();
|
||||
std::string error_message;
|
||||
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message,
|
||||
/* hilite = */ false, "", /* allow_multi_statements = */ false);
|
||||
|
||||
if (!ast && throw_on_error)
|
||||
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
|
||||
|
||||
return ast;
|
||||
}
|
||||
|
||||
ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const
|
||||
{
|
||||
return getCreateTableQueryImpl(context, table_name, true);
|
||||
}
|
||||
|
||||
ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
|
||||
{
|
||||
return getCreateTableQueryImpl(context, table_name, false);
|
||||
}
|
||||
|
||||
ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
|
||||
{
|
||||
String query;
|
||||
{
|
||||
WriteBufferFromString buffer(query);
|
||||
buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary";
|
||||
}
|
||||
ParserCreateQuery parser;
|
||||
return parseQuery(parser, query.data(), query.data() + query.size(), "");
|
||||
}
|
||||
|
||||
void DatabaseDictionary::shutdown()
|
||||
|
||||
@ -22,16 +22,6 @@ class ExternalDictionaries;
|
||||
*/
|
||||
class DatabaseDictionary : public IDatabase
|
||||
{
|
||||
private:
|
||||
const String name;
|
||||
mutable std::mutex mutex;
|
||||
const ExternalDictionaries & external_dictionaries;
|
||||
std::unordered_set<String> deleted_tables;
|
||||
|
||||
Poco::Logger * log;
|
||||
|
||||
Tables loadTables();
|
||||
|
||||
public:
|
||||
DatabaseDictionary(const String & name_, const Context & context);
|
||||
|
||||
@ -79,22 +69,37 @@ public:
|
||||
void alterTable(
|
||||
const Context & context,
|
||||
const String & name,
|
||||
const NamesAndTypesList & columns,
|
||||
const NamesAndTypesList & materialized_columns,
|
||||
const NamesAndTypesList & alias_columns,
|
||||
const ColumnDefaults & column_defaults,
|
||||
const ColumnsDescription & columns,
|
||||
const ASTModifier & engine_modifier) override;
|
||||
|
||||
time_t getTableMetadataModificationTime(
|
||||
const Context & context,
|
||||
const String & table_name) override;
|
||||
|
||||
ASTPtr getCreateQuery(
|
||||
ASTPtr getCreateTableQuery(
|
||||
const Context & context,
|
||||
const String & table_name) const override;
|
||||
|
||||
ASTPtr tryGetCreateTableQuery(
|
||||
const Context & context,
|
||||
const String & table_name) const override;
|
||||
|
||||
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
|
||||
|
||||
void shutdown() override;
|
||||
void drop() override;
|
||||
|
||||
private:
|
||||
const String name;
|
||||
mutable std::mutex mutex;
|
||||
const ExternalDictionaries & external_dictionaries;
|
||||
std::unordered_set<String> deleted_tables;
|
||||
|
||||
Poco::Logger * log;
|
||||
|
||||
Tables loadTables();
|
||||
|
||||
ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -8,75 +8,22 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TABLE_ALREADY_EXISTS;
|
||||
extern const int UNKNOWN_TABLE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
|
||||
}
|
||||
|
||||
DatabaseMemory::DatabaseMemory(String name_)
|
||||
: DatabaseWithOwnTablesBase(std::move(name_))
|
||||
, log(&Logger::get("DatabaseMemory(" + name + ")"))
|
||||
{}
|
||||
|
||||
void DatabaseMemory::loadTables(
|
||||
Context & /*context*/,
|
||||
ThreadPool * /*thread_pool*/,
|
||||
bool /*has_force_restore_data_flag*/)
|
||||
{
|
||||
log = &Logger::get("DatabaseMemory(" + name + ")");
|
||||
|
||||
/// Nothing to load.
|
||||
}
|
||||
|
||||
bool DatabaseMemory::isTableExist(
|
||||
const Context & /*context*/,
|
||||
const String & table_name) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
return tables.find(table_name) != tables.end();
|
||||
}
|
||||
|
||||
StoragePtr DatabaseMemory::tryGetTable(
|
||||
const Context & /*context*/,
|
||||
const String & table_name) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
auto it = tables.find(table_name);
|
||||
if (it == tables.end())
|
||||
return {};
|
||||
return it->second;
|
||||
}
|
||||
|
||||
DatabaseIteratorPtr DatabaseMemory::getIterator(const Context & /*context*/)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
return std::make_unique<DatabaseSnaphotIterator>(tables);
|
||||
}
|
||||
|
||||
bool DatabaseMemory::empty(const Context & /*context*/) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
return tables.empty();
|
||||
}
|
||||
|
||||
StoragePtr DatabaseMemory::detachTable(const String & table_name)
|
||||
{
|
||||
StoragePtr res;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
auto it = tables.find(table_name);
|
||||
if (it == tables.end())
|
||||
throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
|
||||
res = it->second;
|
||||
tables.erase(it);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void DatabaseMemory::attachTable(const String & table_name, const StoragePtr & table)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (!tables.emplace(table_name, table).second)
|
||||
throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
void DatabaseMemory::createTable(
|
||||
const Context & /*context*/,
|
||||
const String & table_name,
|
||||
@ -105,10 +52,7 @@ void DatabaseMemory::renameTable(
|
||||
void DatabaseMemory::alterTable(
|
||||
const Context &,
|
||||
const String &,
|
||||
const NamesAndTypesList &,
|
||||
const NamesAndTypesList &,
|
||||
const NamesAndTypesList &,
|
||||
const ColumnDefaults &,
|
||||
const ColumnsDescription &,
|
||||
const ASTModifier &)
|
||||
{
|
||||
throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
@ -121,31 +65,17 @@ time_t DatabaseMemory::getTableMetadataModificationTime(
|
||||
return static_cast<time_t>(0);
|
||||
}
|
||||
|
||||
ASTPtr DatabaseMemory::getCreateQuery(
|
||||
ASTPtr DatabaseMemory::getCreateTableQuery(
|
||||
const Context &,
|
||||
const String &) const
|
||||
{
|
||||
throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
|
||||
}
|
||||
|
||||
void DatabaseMemory::shutdown()
|
||||
ASTPtr DatabaseMemory::getCreateDatabaseQuery(
|
||||
const Context &) const
|
||||
{
|
||||
/// You can not hold a lock during shutdown.
|
||||
/// Because inside `shutdown` function tables can work with database, and mutex is not recursive.
|
||||
|
||||
Tables tables_snapshot;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
tables_snapshot = tables;
|
||||
}
|
||||
|
||||
for (const auto & kv: tables_snapshot)
|
||||
{
|
||||
kv.second->shutdown();
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
tables.clear();
|
||||
throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
|
||||
}
|
||||
|
||||
void DatabaseMemory::drop()
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <Databases/IDatabase.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
|
||||
|
||||
namespace Poco { class Logger; }
|
||||
@ -16,18 +14,10 @@ namespace DB
|
||||
* All tables are created by calling code.
|
||||
* TODO: Maybe DatabaseRuntime is more suitable class name.
|
||||
*/
|
||||
class DatabaseMemory : public IDatabase
|
||||
class DatabaseMemory : public DatabaseWithOwnTablesBase
|
||||
{
|
||||
protected:
|
||||
const String name;
|
||||
mutable std::mutex mutex;
|
||||
Tables tables;
|
||||
|
||||
Poco::Logger * log;
|
||||
|
||||
public:
|
||||
|
||||
DatabaseMemory(const String & name_) : name(name_) {}
|
||||
DatabaseMemory(String name_);
|
||||
|
||||
String getEngineName() const override { return "Memory"; }
|
||||
|
||||
@ -36,18 +26,6 @@ public:
|
||||
ThreadPool * thread_pool,
|
||||
bool has_force_restore_data_flag) override;
|
||||
|
||||
bool empty(const Context & context) const override;
|
||||
|
||||
DatabaseIteratorPtr getIterator(const Context & context) override;
|
||||
|
||||
bool isTableExist(
|
||||
const Context & context,
|
||||
const String & table_name) const override;
|
||||
|
||||
StoragePtr tryGetTable(
|
||||
const Context & context,
|
||||
const String & table_name) const override;
|
||||
|
||||
void createTable(
|
||||
const Context & context,
|
||||
const String & table_name,
|
||||
@ -58,9 +36,6 @@ public:
|
||||
const Context & context,
|
||||
const String & table_name) override;
|
||||
|
||||
void attachTable(const String & table_name, const StoragePtr & table) override;
|
||||
StoragePtr detachTable(const String & table_name) override;
|
||||
|
||||
void renameTable(
|
||||
const Context & context,
|
||||
const String & table_name,
|
||||
@ -70,22 +45,22 @@ public:
|
||||
void alterTable(
|
||||
const Context & context,
|
||||
const String & name,
|
||||
const NamesAndTypesList & columns,
|
||||
const NamesAndTypesList & materialized_columns,
|
||||
const NamesAndTypesList & alias_columns,
|
||||
const ColumnDefaults & column_defaults,
|
||||
const ColumnsDescription & columns,
|
||||
const ASTModifier & engine_modifier) override;
|
||||
|
||||
time_t getTableMetadataModificationTime(
|
||||
const Context & context,
|
||||
const String & table_name) override;
|
||||
|
||||
ASTPtr getCreateQuery(
|
||||
const Context & context,
|
||||
const String & table_name) const override;
|
||||
ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override;
|
||||
ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; }
|
||||
|
||||
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
|
||||
|
||||
void shutdown() override;
|
||||
void drop() override;
|
||||
|
||||
private:
|
||||
Poco::Logger * log;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user