mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into table-constraints
This commit is contained in:
Gleb Novikov
commit
2b5cf46842
@ -3,7 +3,6 @@
|
||||
### New Features
|
||||
* TTL expressions for columns and tables. [#4212](https://github.com/yandex/ClickHouse/pull/4212) ([Anton Popov](https://github.com/CurtizJ))
|
||||
* Added support for `brotli` compression for HTTP responses (Accept-Encoding: br) [#4388](https://github.com/yandex/ClickHouse/pull/4388) ([Mikhail](https://github.com/fandyushin))
|
||||
* Added a new aggregate function `simpleLinearRegression(x, y)` which performs linear regression on points (x, y) and returns the parameters of the line. (from #4668) [#4917](https://github.com/yandex/ClickHouse/pull/4917) ([hcz](https://github.com/hczhcz))
|
||||
* Added new function `isValidUTF8` for checking whether a set of bytes is correctly utf-8 encoded. [#4934](https://github.com/yandex/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1))
|
||||
* Add new load balancing policy `first_or_random` which sends queries to the first specified host and if it's inaccessible send queries to random hosts of shard. Useful for cross-replication topology setups. [#5012](https://github.com/yandex/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei))
|
||||
|
||||
@ -346,7 +345,7 @@
|
||||
* Added support of `Nullable` types in `mysql` table function. [#4198](https://github.com/yandex/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin))
|
||||
* Support for arbitrary constant expressions in `LIMIT` clause. [#4246](https://github.com/yandex/ClickHouse/pull/4246) ([k3box](https://github.com/k3box))
|
||||
* Added `topKWeighted` aggregate function that takes additional argument with (unsigned integer) weight. [#4245](https://github.com/yandex/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman))
|
||||
* `StorageJoin` now supports `join_overwrite` setting that allows overwriting existing values of the same key. [#3973](https://github.com/yandex/ClickHouse/pull/3973) ([Amos Bird](https://github.com/amosbird)
|
||||
* `StorageJoin` now supports `join_any_take_last_row` setting that allows overwriting existing values of the same key. [#3973](https://github.com/yandex/ClickHouse/pull/3973) ([Amos Bird](https://github.com/amosbird)
|
||||
* Added function `toStartOfInterval`. [#4304](https://github.com/yandex/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar))
|
||||
* Added `RowBinaryWithNamesAndTypes` format. [#4200](https://github.com/yandex/ClickHouse/pull/4200) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer))
|
||||
* Added `IPv4` and `IPv6` data types. More effective implementations of `IPv*` functions. [#3669](https://github.com/yandex/ClickHouse/pull/3669) ([Vasily Nemkov](https://github.com/Enmk))
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
### Новые возможности
|
||||
* TTL выражения, позволяющие настроить время жизни и автоматическую очистку данных в таблице или в отдельных её столбцах. [#4212](https://github.com/yandex/ClickHouse/pull/4212) ([Anton Popov](https://github.com/CurtizJ))
|
||||
* Добавлена поддержка алгоритма сжатия `brotli` в HTTP ответах (`Accept-Encoding: br`). Для тела POST запросов, эта возможность уже существовала. [#4388](https://github.com/yandex/ClickHouse/pull/4388) ([Mikhail](https://github.com/fandyushin))
|
||||
* Добавлена агрегатная функция `simpleLinearRegression(x, y)` которая осуществляет линейную регрессию на точках (x, y) и возвращает параметры линии. (from #4668) [#4917](https://github.com/yandex/ClickHouse/pull/4917) ([hcz](https://github.com/hczhcz))
|
||||
* Добавлена функция `isValidUTF8` для проверки, содержит ли строка валидные данные в кодировке UTF-8. [#4934](https://github.com/yandex/ClickHouse/pull/4934) ([Danila Kutenin](https://github.com/danlark1))
|
||||
* Добавлены новое правило балансировки (`load_balancing`) `first_or_random` по которому запросы посылаются на первый заданый хост и если он недоступен - на случайные хосты шарда. Полезно для топологий с кросс-репликацией. [#5012](https://github.com/yandex/ClickHouse/pull/5012) ([nvartolomei](https://github.com/nvartolomei))
|
||||
|
||||
@ -337,7 +336,7 @@
|
||||
* Добавлена поддержка `Nullable` типов в табличной функции `mysql`. [#4198](https://github.com/yandex/ClickHouse/pull/4198) ([Emmanuel Donin de Rosière](https://github.com/edonin))
|
||||
* Добавлена поддержка произвольных константных выражений в секции `LIMIT`. [#4246](https://github.com/yandex/ClickHouse/pull/4246) ([k3box](https://github.com/k3box))
|
||||
* Добавлена агрегатная функция `topKWeighted` - вариант `topK`, позволяющий задавать (целый неотрицательный) вес добавляемого значения. [#4245](https://github.com/yandex/ClickHouse/pull/4245) ([Andrew Golman](https://github.com/andrewgolman))
|
||||
* Движок `Join` теперь поддерживает настройку `join_overwrite`, которая позволяет перезаписывать значения для существующих ключей. [#3973](https://github.com/yandex/ClickHouse/pull/3973) ([Amos Bird](https://github.com/amosbird))
|
||||
* Движок `Join` теперь поддерживает настройку `join_any_take_last_row`, которая позволяет перезаписывать значения для существующих ключей. [#3973](https://github.com/yandex/ClickHouse/pull/3973) ([Amos Bird](https://github.com/amosbird))
|
||||
* Добавлена функция `toStartOfInterval`. [#4304](https://github.com/yandex/ClickHouse/pull/4304) ([Vitaly Baranov](https://github.com/vitlibar))
|
||||
* Добавлена функция `toStartOfTenMinutes`. [#4298](https://github.com/yandex/ClickHouse/pull/4298) ([Vitaly Baranov](https://github.com/vitlibar))
|
||||
* Добавлен формат `RowBinaryWithNamesAndTypes`. [#4200](https://github.com/yandex/ClickHouse/pull/4200) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer))
|
||||
|
||||
@ -93,6 +93,11 @@ if (COMPILER_GCC OR COMPILER_CLANG)
|
||||
set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wnon-virtual-dtor")
|
||||
endif ()
|
||||
|
||||
if (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "8.3.0")
|
||||
# Warnings in protobuf generating
|
||||
set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wno-array-bounds")
|
||||
endif ()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
# clang: warning: argument unused during compilation: '-stdlib=libc++'
|
||||
# clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
|
||||
|
||||
4
contrib/CMakeLists.txt
vendored
4
contrib/CMakeLists.txt
vendored
@ -1,8 +1,8 @@
|
||||
# Third-party libraries may have substandard code.
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch -Wno-stringop-truncation")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -Wno-array-bounds -Wno-missing-attributes -Wno-stringop-truncation -std=c++1z")
|
||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch -Wno-string-plus-int")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -Wno-inconsistent-missing-override -std=c++1z")
|
||||
|
||||
2
contrib/boost
vendored
2
contrib/boost
vendored
@ -1 +1 @@
|
||||
Subproject commit 79bf85ea99c05ba4fb6959474d4464ab126f8973
|
||||
Subproject commit 8abda007bfe52d78a51548d4594879d6d82a22fa
|
||||
2
contrib/hyperscan
vendored
2
contrib/hyperscan
vendored
@ -1 +1 @@
|
||||
Subproject commit 05b0f9064cca4bd55548dedb0a32ed9461146c1e
|
||||
Subproject commit ed17d34a7c786512471946f9105eaa8d925f34c3
|
||||
@ -40,6 +40,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
list(APPEND SRCS ${JEMALLOC_SOURCE_DIR}/src/zone.c)
|
||||
endif()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
|
||||
endif ()
|
||||
|
||||
add_library(jemalloc STATIC ${SRCS})
|
||||
|
||||
target_include_directories(jemalloc PUBLIC
|
||||
|
||||
@ -49,19 +49,20 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
endif ()
|
||||
|
||||
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
|
||||
if (WEVERYTHING)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wshadow-field -Wstring-plus-int -Wempty-init-stmt")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow-field -Wstring-plus-int")
|
||||
if(NOT APPLE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wempty-init-stmt")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
|
||||
if (WEVERYTHING)
|
||||
if (WEVERYTHING AND NOT APPLE)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ctad-maybe-unsupported")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
set(VERSION_REVISION 54420)
|
||||
set(VERSION_REVISION 54421)
|
||||
set(VERSION_MAJOR 19)
|
||||
set(VERSION_MINOR 8)
|
||||
set(VERSION_MINOR 9)
|
||||
set(VERSION_PATCH 1)
|
||||
set(VERSION_GITHASH a76e504f45ff4a74e8c492bd269f022352d5f6d9)
|
||||
set(VERSION_DESCRIBE v19.8.1.1-testing)
|
||||
set(VERSION_STRING 19.8.1.1)
|
||||
set(VERSION_GITHASH 0c2aa460651a462f14efc7e995840a244531d373)
|
||||
set(VERSION_DESCRIBE v19.9.1.1-testing)
|
||||
set(VERSION_STRING 19.9.1.1)
|
||||
# end of autochange
|
||||
|
||||
set(VERSION_EXTRA "" CACHE STRING "")
|
||||
|
||||
@ -435,7 +435,7 @@ private:
|
||||
#if USE_READLINE
|
||||
int res = read_history(history_file.c_str());
|
||||
if (res)
|
||||
throwFromErrno("Cannot read history from file " + history_file, ErrorCodes::CANNOT_READ_HISTORY);
|
||||
std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY);
|
||||
#endif
|
||||
}
|
||||
else /// Create history file.
|
||||
@ -612,7 +612,7 @@ private:
|
||||
|
||||
#if USE_READLINE && HAVE_READLINE_HISTORY
|
||||
if (!history_file.empty() && append_history(1, history_file.c_str()))
|
||||
throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY);
|
||||
std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY);
|
||||
#endif
|
||||
|
||||
prev_input = input;
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include "PerformanceTest.h"
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/CpuId.h>
|
||||
#include <common/getMemoryAmount.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
@ -71,6 +72,7 @@ bool PerformanceTest::checkPreconditions() const
|
||||
Strings preconditions;
|
||||
config->keys("preconditions", preconditions);
|
||||
size_t table_precondition_index = 0;
|
||||
size_t cpu_precondition_index = 0;
|
||||
|
||||
for (const std::string & precondition : preconditions)
|
||||
{
|
||||
@ -136,6 +138,30 @@ bool PerformanceTest::checkPreconditions() const
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (precondition == "cpu")
|
||||
{
|
||||
std::string precondition_key = "preconditions.cpu[" + std::to_string(cpu_precondition_index++) + "]";
|
||||
std::string flag_to_check = config->getString(precondition_key);
|
||||
|
||||
#define CHECK_CPU_PRECONDITION(OP) \
|
||||
if (flag_to_check == #OP) \
|
||||
{ \
|
||||
if (!Cpu::CpuFlagsCache::have_##OP) \
|
||||
{ \
|
||||
LOG_WARNING(log, "CPU doesn't support " << #OP); \
|
||||
return false; \
|
||||
} \
|
||||
} else
|
||||
|
||||
CPU_ID_ENUMERATE(CHECK_CPU_PRECONDITION)
|
||||
{
|
||||
LOG_WARNING(log, "CPU doesn't support " << flag_to_check);
|
||||
return false;
|
||||
}
|
||||
|
||||
#undef CHECK_CPU_PRECONDITION
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@ -9,6 +9,7 @@ set(CLICKHOUSE_SERVER_SOURCES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp
|
||||
)
|
||||
|
||||
set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
|
||||
|
||||
@ -453,30 +453,6 @@ void HTTPHandler::processQuery(
|
||||
return false;
|
||||
};
|
||||
|
||||
/// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
|
||||
std::string full_query;
|
||||
|
||||
/// Support for "external data for query processing".
|
||||
if (startsWith(request.getContentType().data(), "multipart/form-data"))
|
||||
{
|
||||
ExternalTablesHandler handler(context, params);
|
||||
params.load(request, istr, handler);
|
||||
|
||||
/// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
|
||||
reserved_param_suffixes.emplace_back("_format");
|
||||
reserved_param_suffixes.emplace_back("_types");
|
||||
reserved_param_suffixes.emplace_back("_structure");
|
||||
|
||||
/// Params are of both form params POST and uri (GET params)
|
||||
for (const auto & it : params)
|
||||
if (it.first == "query")
|
||||
full_query += it.second;
|
||||
|
||||
in = std::make_unique<ReadBufferFromString>(full_query);
|
||||
}
|
||||
else
|
||||
in = std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
|
||||
|
||||
/// Settings can be overridden in the query.
|
||||
/// Some parameters (database, default_format, everything used in the code above) do not
|
||||
/// belong to the Settings class.
|
||||
@ -497,30 +473,63 @@ void HTTPHandler::processQuery(
|
||||
settings.readonly = 2;
|
||||
}
|
||||
|
||||
SettingsChanges settings_changes;
|
||||
for (auto it = params.begin(); it != params.end(); ++it)
|
||||
bool isExternalData = startsWith(request.getContentType().data(), "multipart/form-data");
|
||||
|
||||
if (isExternalData)
|
||||
{
|
||||
if (it->first == "database")
|
||||
/// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
|
||||
reserved_param_suffixes.reserve(3);
|
||||
/// It is a bug and ambiguity with `date_time_input_format` and `low_cardinality_allow_in_native_format` formats/settings.
|
||||
reserved_param_suffixes.emplace_back("_format");
|
||||
reserved_param_suffixes.emplace_back("_types");
|
||||
reserved_param_suffixes.emplace_back("_structure");
|
||||
}
|
||||
|
||||
SettingsChanges settings_changes;
|
||||
for (const auto & [key, value] : params)
|
||||
{
|
||||
if (key == "database")
|
||||
{
|
||||
context.setCurrentDatabase(it->second);
|
||||
context.setCurrentDatabase(value);
|
||||
}
|
||||
else if (it->first == "default_format")
|
||||
else if (key == "default_format")
|
||||
{
|
||||
context.setDefaultFormat(it->second);
|
||||
context.setDefaultFormat(value);
|
||||
}
|
||||
else if (param_could_be_skipped(it->first))
|
||||
else if (param_could_be_skipped(key))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
/// All other query parameters are treated as settings.
|
||||
settings_changes.push_back({it->first, it->second});
|
||||
settings_changes.push_back({key, value});
|
||||
}
|
||||
}
|
||||
|
||||
/// For external data we also want settings
|
||||
context.checkSettingsConstraints(settings_changes);
|
||||
context.applySettingsChanges(settings_changes);
|
||||
|
||||
/// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
|
||||
std::string full_query;
|
||||
|
||||
/// Support for "external data for query processing".
|
||||
if (isExternalData)
|
||||
{
|
||||
ExternalTablesHandler handler(context, params);
|
||||
params.load(request, istr, handler);
|
||||
|
||||
/// Params are of both form params POST and uri (GET params)
|
||||
for (const auto & it : params)
|
||||
if (it.first == "query")
|
||||
full_query += it.second;
|
||||
|
||||
in = std::make_unique<ReadBufferFromString>(full_query);
|
||||
}
|
||||
else
|
||||
in = std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
|
||||
|
||||
|
||||
/// HTTP response compression is turned on only if the client signalled that they support it
|
||||
/// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on.
|
||||
used_output.out->setCompression(client_supports_http_compression && settings.enable_http_compression);
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <DataStreams/copyData.h>
|
||||
#include <IO/ReadBufferFromMemory.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/ReadBufferFromPocoSocket.h>
|
||||
#include <IO/WriteBufferFromPocoSocket.h>
|
||||
#include <Interpreters/executeQuery.h>
|
||||
@ -9,12 +9,14 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Poco/Crypto/RSAKey.h>
|
||||
#include <Poco/Crypto/CipherFactory.h>
|
||||
#include <Poco/Net/SecureStreamSocket.h>
|
||||
#include <Poco/Net/SSLManager.h>
|
||||
#include "MySQLHandler.h"
|
||||
#include <limits>
|
||||
#include <ext/scope_guard.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -25,21 +27,32 @@ using Poco::Net::SSLManager;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES;
|
||||
extern const int OPENSSL_ERROR;
|
||||
}
|
||||
|
||||
uint32_t MySQLHandler::last_connection_id = 0;
|
||||
|
||||
MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key, RSA & private_key, bool ssl_enabled, size_t connection_id)
|
||||
: Poco::Net::TCPServerConnection(socket_)
|
||||
, server(server_)
|
||||
, log(&Poco::Logger::get("MySQLHandler"))
|
||||
, connection_context(server.context())
|
||||
, connection_id(connection_id)
|
||||
, public_key(public_key)
|
||||
, private_key(private_key)
|
||||
{
|
||||
server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF;
|
||||
if (ssl_enabled)
|
||||
server_capability_flags |= CLIENT_SSL;
|
||||
}
|
||||
|
||||
void MySQLHandler::run()
|
||||
{
|
||||
connection_context = server.context();
|
||||
connection_context.setDefaultFormat("MySQL");
|
||||
connection_context.setDefaultFormat("MySQLWire");
|
||||
|
||||
in = std::make_shared<ReadBufferFromPocoSocket>(socket());
|
||||
out = std::make_shared<WriteBufferFromPocoSocket>(socket());
|
||||
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.sequence_id, "MySQLHandler");
|
||||
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.sequence_id);
|
||||
|
||||
try
|
||||
{
|
||||
@ -49,8 +62,7 @@ void MySQLHandler::run()
|
||||
* This plugin must do the same to stay consistent with historical behavior if it is set to operate as a default plugin.
|
||||
* https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L3994
|
||||
*/
|
||||
Handshake handshake(connection_id, VERSION_STRING, scramble + '\0');
|
||||
|
||||
Handshake handshake(server_capability_flags, connection_id, VERSION_STRING + String("-") + VERSION_NAME, scramble + '\0');
|
||||
packet_sender->sendPacket<Handshake>(handshake, true);
|
||||
|
||||
LOG_TRACE(log, "Sent handshake");
|
||||
@ -78,15 +90,11 @@ void MySQLHandler::run()
|
||||
<< "\nauth_plugin_name: "
|
||||
<< handshake_response.auth_plugin_name);
|
||||
|
||||
capabilities = handshake_response.capability_flags;
|
||||
if (!(capabilities & CLIENT_PROTOCOL_41))
|
||||
{
|
||||
client_capability_flags = handshake_response.capability_flags;
|
||||
if (!(client_capability_flags & CLIENT_PROTOCOL_41))
|
||||
throw Exception("Required capability: CLIENT_PROTOCOL_41.", ErrorCodes::MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES);
|
||||
}
|
||||
if (!(capabilities & CLIENT_PLUGIN_AUTH))
|
||||
{
|
||||
if (!(client_capability_flags & CLIENT_PLUGIN_AUTH))
|
||||
throw Exception("Required capability: CLIENT_PLUGIN_AUTH.", ErrorCodes::MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES);
|
||||
}
|
||||
|
||||
authenticate(handshake_response, scramble);
|
||||
OK_Packet ok_packet(0, handshake_response.capability_flags, 0, 0, 0);
|
||||
@ -165,7 +173,7 @@ MySQLProtocol::HandshakeResponse MySQLHandler::finishHandshake()
|
||||
};
|
||||
read_bytes(3); /// We can find out whether it is SSLRequest of HandshakeResponse by first 3 bytes.
|
||||
|
||||
size_t payload_size = *reinterpret_cast<uint32_t *>(buf) & 0xFFFFFFu;
|
||||
size_t payload_size = unalignedLoad<uint32_t>(buf) & 0xFFFFFFu;
|
||||
LOG_TRACE(log, "payload size: " << payload_size);
|
||||
|
||||
if (payload_size == SSL_REQUEST_PAYLOAD_SIZE)
|
||||
@ -180,7 +188,7 @@ MySQLProtocol::HandshakeResponse MySQLHandler::finishHandshake()
|
||||
in = std::make_shared<ReadBufferFromPocoSocket>(*ss);
|
||||
out = std::make_shared<WriteBufferFromPocoSocket>(*ss);
|
||||
connection_context.sequence_id = 2;
|
||||
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.sequence_id, "MySQLHandler");
|
||||
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.sequence_id);
|
||||
packet_sender->max_packet_size = connection_context.max_packet_size;
|
||||
packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket.
|
||||
}
|
||||
@ -216,6 +224,10 @@ void MySQLHandler::authenticate(const HandshakeResponse & handshake_response, co
|
||||
if (handshake_response.auth_plugin_name != Authentication::SHA256)
|
||||
{
|
||||
packet_sender->sendPacket(AuthSwitchRequest(Authentication::SHA256, scramble + '\0'), true);
|
||||
if (in->eof())
|
||||
throw Exception(
|
||||
"Client doesn't support authentication method " + String(Authentication::SHA256) + " used by ClickHouse",
|
||||
ErrorCodes::MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES);
|
||||
packet_sender->receivePacket(response);
|
||||
auth_response = response.value;
|
||||
LOG_TRACE(log, "Authentication method mismatch.");
|
||||
@ -226,31 +238,19 @@ void MySQLHandler::authenticate(const HandshakeResponse & handshake_response, co
|
||||
LOG_TRACE(log, "Authentication method match.");
|
||||
}
|
||||
|
||||
auto getOpenSSLError = []() -> String
|
||||
{
|
||||
BIO * mem = BIO_new(BIO_s_mem());
|
||||
ERR_print_errors(mem);
|
||||
char * buf = nullptr;
|
||||
long size = BIO_get_mem_data(mem, &buf);
|
||||
String errors_str(buf, size);
|
||||
BIO_free(mem);
|
||||
return errors_str;
|
||||
};
|
||||
|
||||
if (auth_response == "\1")
|
||||
{
|
||||
LOG_TRACE(log, "Client requests public key.");
|
||||
|
||||
BIO * mem = BIO_new(BIO_s_mem());
|
||||
if (PEM_write_bio_RSA_PUBKEY(mem, public_key) != 1)
|
||||
SCOPE_EXIT(BIO_free(mem));
|
||||
if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1)
|
||||
{
|
||||
LOG_TRACE(log, "OpenSSL error:\n" << getOpenSSLError());
|
||||
throw Exception("Failed to write public key to memory.", ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
throw Exception("Failed to write public key to memory. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
char * pem_buf = nullptr;
|
||||
long pem_size = BIO_get_mem_data(mem, &pem_buf);
|
||||
String pem(pem_buf, pem_size);
|
||||
BIO_free(mem);
|
||||
|
||||
LOG_TRACE(log, "Key: " << pem);
|
||||
|
||||
@ -271,17 +271,16 @@ void MySQLHandler::authenticate(const HandshakeResponse & handshake_response, co
|
||||
* an empty packet is a blank password, thus the check for auth_response.empty() has to be made too.
|
||||
* https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L4017
|
||||
*/
|
||||
if (!secure_connection && (!auth_response.empty() && auth_response != "\0"))
|
||||
if (!secure_connection && !auth_response.empty() && auth_response != String("\0", 1))
|
||||
{
|
||||
LOG_TRACE(log, "Received nonempty password");
|
||||
auto ciphertext = reinterpret_cast<unsigned char *>(auth_response.data());
|
||||
|
||||
unsigned char plaintext[RSA_size(private_key)];
|
||||
int plaintext_size = RSA_private_decrypt(auth_response.size(), ciphertext, plaintext, private_key, RSA_PKCS1_OAEP_PADDING);
|
||||
unsigned char plaintext[RSA_size(&private_key)];
|
||||
int plaintext_size = RSA_private_decrypt(auth_response.size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING);
|
||||
if (plaintext_size == -1)
|
||||
{
|
||||
LOG_TRACE(log, "OpenSSL error:\n" << getOpenSSLError());
|
||||
throw Exception("Failed to decrypt.", ErrorCodes::UNKNOWN_EXCEPTION);
|
||||
throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
|
||||
password.resize(plaintext_size);
|
||||
@ -324,7 +323,7 @@ void MySQLHandler::comInitDB(const String & payload)
|
||||
String database = payload.substr(1);
|
||||
LOG_DEBUG(log, "Setting current database to " << database);
|
||||
connection_context.setCurrentDatabase(database);
|
||||
packet_sender->sendPacket(OK_Packet(0, capabilities, 0, 0, 1), true);
|
||||
packet_sender->sendPacket(OK_Packet(0, client_capability_flags, 0, 0, 1), true);
|
||||
}
|
||||
|
||||
void MySQLHandler::comFieldList(const String & payload)
|
||||
@ -340,12 +339,12 @@ void MySQLHandler::comFieldList(const String & payload)
|
||||
);
|
||||
packet_sender->sendPacket(column_definition);
|
||||
}
|
||||
packet_sender->sendPacket(OK_Packet(0xfe, capabilities, 0, 0, 0), true);
|
||||
packet_sender->sendPacket(OK_Packet(0xfe, client_capability_flags, 0, 0, 0), true);
|
||||
}
|
||||
|
||||
void MySQLHandler::comPing()
|
||||
{
|
||||
packet_sender->sendPacket(OK_Packet(0x0, capabilities, 0, 0, 0), true);
|
||||
packet_sender->sendPacket(OK_Packet(0x0, client_capability_flags, 0, 0, 0), true);
|
||||
}
|
||||
|
||||
void MySQLHandler::comQuery(const String & payload)
|
||||
@ -354,10 +353,18 @@ void MySQLHandler::comQuery(const String & payload)
|
||||
std::function<void(const String &)> set_content_type = [&with_output](const String &) -> void {
|
||||
with_output = true;
|
||||
};
|
||||
ReadBufferFromMemory query(payload.data() + 1, payload.size() - 1);
|
||||
executeQuery(query, *out, true, connection_context, set_content_type, nullptr);
|
||||
|
||||
String query = payload.substr(1);
|
||||
|
||||
// Translate query from MySQL to ClickHouse.
|
||||
// This is a temporary workaround until ClickHouse supports the syntax "@@var_name".
|
||||
if (query == "select @@version_comment limit 1") // MariaDB client starts session with that query
|
||||
query = "select ''";
|
||||
|
||||
ReadBufferFromString buf(query);
|
||||
executeQuery(buf, *out, true, connection_context, set_content_type, nullptr);
|
||||
if (!with_output)
|
||||
packet_sender->sendPacket(OK_Packet(0x00, capabilities, 0, 0, 0), true);
|
||||
packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Poco/Net/SecureStreamSocket.h>
|
||||
#include <Common/getFQDNOrHostName.h>
|
||||
#include <Core/MySQLProtocol.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/rsa.h>
|
||||
#include "IServer.h"
|
||||
|
||||
|
||||
@ -15,21 +15,7 @@ namespace DB
|
||||
class MySQLHandler : public Poco::Net::TCPServerConnection
|
||||
{
|
||||
public:
|
||||
MySQLHandler(
|
||||
IServer & server_,
|
||||
const Poco::Net::StreamSocket & socket_,
|
||||
RSA * public_key,
|
||||
RSA * private_key)
|
||||
: Poco::Net::TCPServerConnection(socket_)
|
||||
, server(server_)
|
||||
, log(&Poco::Logger::get("MySQLHandler"))
|
||||
, connection_context(server.context())
|
||||
, connection_id(last_connection_id++)
|
||||
, public_key(public_key)
|
||||
, private_key(private_key)
|
||||
{
|
||||
log->setLevel("information");
|
||||
}
|
||||
MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key, RSA & private_key, bool ssl_enabled, size_t connection_id);
|
||||
|
||||
void run() final;
|
||||
|
||||
@ -55,13 +41,13 @@ private:
|
||||
|
||||
std::shared_ptr<MySQLProtocol::PacketSender> packet_sender;
|
||||
|
||||
uint32_t connection_id = 0;
|
||||
size_t connection_id = 0;
|
||||
|
||||
uint32_t capabilities;
|
||||
size_t server_capability_flags;
|
||||
size_t client_capability_flags;
|
||||
|
||||
static uint32_t last_connection_id;
|
||||
|
||||
RSA * public_key, * private_key;
|
||||
RSA & public_key;
|
||||
RSA & private_key;
|
||||
|
||||
std::shared_ptr<ReadBuffer> in;
|
||||
std::shared_ptr<WriteBuffer> out;
|
||||
|
||||
124
dbms/programs/server/MySQLHandlerFactory.cpp
Normal file
124
dbms/programs/server/MySQLHandlerFactory.cpp
Normal file
@ -0,0 +1,124 @@
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Poco/Crypto/X509Certificate.h>
|
||||
#include <Poco/Net/SSLManager.h>
|
||||
#include <Poco/Net/TCPServerConnectionFactory.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include "IServer.h"
|
||||
#include "MySQLHandler.h"
|
||||
#include "MySQLHandlerFactory.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int NO_ELEMENTS_IN_CONFIG;
|
||||
extern const int OPENSSL_ERROR;
|
||||
extern const int SYSTEM_ERROR;
|
||||
}
|
||||
|
||||
MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
|
||||
: server(server_)
|
||||
, log(&Logger::get("MySQLHandlerFactory"))
|
||||
{
|
||||
try
|
||||
{
|
||||
Poco::Net::SSLManager::instance().defaultServerContext();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
|
||||
ssl_enabled = false;
|
||||
}
|
||||
|
||||
/// Reading rsa keys for SHA256 authentication plugin.
|
||||
try
|
||||
{
|
||||
readRSAKeys();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false));
|
||||
generateRSAKeys();
|
||||
}
|
||||
}
|
||||
|
||||
void MySQLHandlerFactory::readRSAKeys()
|
||||
{
|
||||
const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
|
||||
String certificateFileProperty = "openSSL.server.certificateFile";
|
||||
String privateKeyFileProperty = "openSSL.server.privateKeyFile";
|
||||
|
||||
if (!config.has(certificateFileProperty))
|
||||
throw Exception("Certificate file is not set.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
|
||||
|
||||
if (!config.has(privateKeyFileProperty))
|
||||
throw Exception("Private key file is not set.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
|
||||
|
||||
{
|
||||
String certificateFile = config.getString(certificateFileProperty);
|
||||
FILE * fp = fopen(certificateFile.data(), "r");
|
||||
if (fp == nullptr)
|
||||
throw Exception("Cannot open certificate file: " + certificateFile + ".", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
SCOPE_EXIT(fclose(fp));
|
||||
|
||||
X509 * x509 = PEM_read_X509(fp, nullptr, nullptr, nullptr);
|
||||
SCOPE_EXIT(X509_free(x509));
|
||||
if (x509 == nullptr)
|
||||
throw Exception("Failed to read PEM certificate from " + certificateFile + ". Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
EVP_PKEY * p = X509_get_pubkey(x509);
|
||||
if (p == nullptr)
|
||||
throw Exception("Failed to get RSA key from X509. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
SCOPE_EXIT(EVP_PKEY_free(p));
|
||||
|
||||
public_key.reset(EVP_PKEY_get1_RSA(p));
|
||||
if (public_key.get() == nullptr)
|
||||
throw Exception("Failed to get RSA key from ENV_PKEY. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
|
||||
{
|
||||
String privateKeyFile = config.getString(privateKeyFileProperty);
|
||||
|
||||
FILE * fp = fopen(privateKeyFile.data(), "r");
|
||||
if (fp == nullptr)
|
||||
throw Exception ("Cannot open private key file " + privateKeyFile + ".", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
SCOPE_EXIT(fclose(fp));
|
||||
|
||||
private_key.reset(PEM_read_RSAPrivateKey(fp, nullptr, nullptr, nullptr));
|
||||
if (!private_key)
|
||||
throw Exception("Failed to read RSA private key from " + privateKeyFile + ". Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
void MySQLHandlerFactory::generateRSAKeys()
|
||||
{
|
||||
LOG_INFO(log, "Generating new RSA key.");
|
||||
public_key.reset(RSA_new());
|
||||
if (!public_key)
|
||||
throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
BIGNUM * e = BN_new();
|
||||
if (!e)
|
||||
throw Exception("Failed to allocate BIGNUM. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
SCOPE_EXIT(BN_free(e));
|
||||
|
||||
if (!BN_set_word(e, 65537) || !RSA_generate_key_ex(public_key.get(), 2048, e, nullptr))
|
||||
throw Exception("Failed to generate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
|
||||
private_key.reset(RSAPrivateKey_dup(public_key.get()));
|
||||
if (!private_key)
|
||||
throw Exception("Failed to copy RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
|
||||
}
|
||||
|
||||
Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket)
|
||||
{
|
||||
size_t connection_id = last_connection_id++;
|
||||
LOG_TRACE(log, "MySQL connection. Id: " << connection_id << ". Address: " << socket.peerAddress().toString());
|
||||
return new MySQLHandler(server, socket, *public_key, *private_key, ssl_enabled, connection_id);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,16 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Net/TCPServerConnectionFactory.h>
|
||||
#include <Poco/Net/SSLManager.h>
|
||||
#include <Poco/Crypto/X509Certificate.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <atomic>
|
||||
#include <openssl/rsa.h>
|
||||
#include "IServer.h"
|
||||
#include "MySQLHandler.h"
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
class Logger;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -20,94 +13,27 @@ class MySQLHandlerFactory : public Poco::Net::TCPServerConnectionFactory
|
||||
private:
|
||||
IServer & server;
|
||||
Poco::Logger * log;
|
||||
RSA * public_key = nullptr, * private_key = nullptr;
|
||||
|
||||
struct RSADeleter
|
||||
{
|
||||
void operator()(RSA * ptr) { RSA_free(ptr); }
|
||||
};
|
||||
using RSAPtr = std::unique_ptr<RSA, RSADeleter>;
|
||||
|
||||
RSAPtr public_key;
|
||||
RSAPtr private_key;
|
||||
|
||||
bool ssl_enabled = true;
|
||||
|
||||
std::atomic<size_t> last_connection_id = 0;
|
||||
public:
|
||||
explicit MySQLHandlerFactory(IServer & server_)
|
||||
: server(server_), log(&Logger::get("MySQLHandlerFactory"))
|
||||
{
|
||||
/// Reading rsa keys for SHA256 authentication plugin.
|
||||
const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
|
||||
String certificateFileProperty = "openSSL.server.certificateFile";
|
||||
String privateKeyFileProperty = "openSSL.server.privateKeyFile";
|
||||
explicit MySQLHandlerFactory(IServer & server_);
|
||||
|
||||
if (!config.has(certificateFileProperty))
|
||||
{
|
||||
LOG_INFO(log, "Certificate file is not set.");
|
||||
generateRSAKeys();
|
||||
return;
|
||||
}
|
||||
if (!config.has(privateKeyFileProperty))
|
||||
{
|
||||
LOG_INFO(log, "Private key file is not set.");
|
||||
generateRSAKeys();
|
||||
return;
|
||||
}
|
||||
void readRSAKeys();
|
||||
|
||||
String certificateFile = config.getString(certificateFileProperty);
|
||||
FILE * fp = fopen(certificateFile.data(), "r");
|
||||
if (fp == nullptr)
|
||||
{
|
||||
LOG_WARNING(log, "Cannot open certificate file: " << certificateFile << ".");
|
||||
generateRSAKeys();
|
||||
return;
|
||||
}
|
||||
X509 * x509 = PEM_read_X509(fp, nullptr, nullptr, nullptr);
|
||||
EVP_PKEY * p = X509_get_pubkey(x509);
|
||||
public_key = EVP_PKEY_get1_RSA(p);
|
||||
X509_free(x509);
|
||||
EVP_PKEY_free(p);
|
||||
fclose(fp);
|
||||
void generateRSAKeys();
|
||||
|
||||
String privateKeyFile = config.getString(privateKeyFileProperty);
|
||||
fp = fopen(privateKeyFile.data(), "r");
|
||||
if (fp == nullptr)
|
||||
{
|
||||
LOG_WARNING(log, "Cannot open private key file " << privateKeyFile << ".");
|
||||
generateRSAKeys();
|
||||
return;
|
||||
}
|
||||
private_key = PEM_read_RSAPrivateKey(fp, nullptr, nullptr, nullptr);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
void generateRSAKeys()
|
||||
{
|
||||
LOG_INFO(log, "Generating new RSA key.");
|
||||
RSA * rsa = RSA_new();
|
||||
if (rsa == nullptr)
|
||||
{
|
||||
throw Exception("Failed to allocate RSA key.", 1002);
|
||||
}
|
||||
BIGNUM * e = BN_new();
|
||||
if (!e)
|
||||
{
|
||||
RSA_free(rsa);
|
||||
throw Exception("Failed to allocate BIGNUM.", 1002);
|
||||
}
|
||||
if (!BN_set_word(e, 65537) || !RSA_generate_key_ex(rsa, 2048, e, nullptr))
|
||||
{
|
||||
RSA_free(rsa);
|
||||
BN_free(e);
|
||||
throw Exception("Failed to generate RSA key.", 1002);
|
||||
}
|
||||
BN_free(e);
|
||||
|
||||
public_key = rsa;
|
||||
private_key = RSAPrivateKey_dup(rsa);
|
||||
}
|
||||
|
||||
Poco::Net::TCPServerConnection * createConnection(const Poco::Net::StreamSocket & socket) override
|
||||
{
|
||||
LOG_TRACE(log, "MySQL connection. Address: " << socket.peerAddress().toString());
|
||||
return new MySQLHandler(server, socket, public_key, private_key);
|
||||
}
|
||||
|
||||
~MySQLHandlerFactory() override
|
||||
{
|
||||
RSA_free(public_key);
|
||||
RSA_free(private_key);
|
||||
}
|
||||
Poco::Net::TCPServerConnection * createConnection(const Poco::Net::StreamSocket & socket) override;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include "AggregateFunctionFactory.h"
|
||||
@ -43,11 +45,11 @@ namespace
|
||||
|
||||
/// Such default parameters were picked because they did good on some tests,
|
||||
/// though it still requires to fit parameters to achieve better result
|
||||
auto learning_rate = Float64(0.01);
|
||||
auto l2_reg_coef = Float64(0.01);
|
||||
UInt32 batch_size = 1;
|
||||
auto learning_rate = Float64(0.00001);
|
||||
auto l2_reg_coef = Float64(0.1);
|
||||
UInt32 batch_size = 15;
|
||||
|
||||
std::shared_ptr<IWeightsUpdater> weights_updater = std::make_shared<StochasticGradientDescent>();
|
||||
std::string weights_updater_name = "\'SGD\'";
|
||||
std::shared_ptr<IGradientComputer> gradient_computer;
|
||||
|
||||
if (!parameters.empty())
|
||||
@ -64,19 +66,8 @@ namespace
|
||||
}
|
||||
if (parameters.size() > 3)
|
||||
{
|
||||
if (applyVisitor(FieldVisitorToString(), parameters[3]) == "\'SGD\'")
|
||||
{
|
||||
weights_updater = std::make_shared<StochasticGradientDescent>();
|
||||
}
|
||||
else if (applyVisitor(FieldVisitorToString(), parameters[3]) == "\'Momentum\'")
|
||||
{
|
||||
weights_updater = std::make_shared<Momentum>();
|
||||
}
|
||||
else if (applyVisitor(FieldVisitorToString(), parameters[3]) == "\'Nesterov\'")
|
||||
{
|
||||
weights_updater = std::make_shared<Nesterov>();
|
||||
}
|
||||
else
|
||||
weights_updater_name = applyVisitor(FieldVisitorToString(), parameters[3]);
|
||||
if (weights_updater_name != "\'SGD\'" && weights_updater_name != "\'Momentum\'" && weights_updater_name != "\'Nesterov\'")
|
||||
{
|
||||
throw Exception("Invalid parameter for weights updater", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
@ -98,20 +89,19 @@ namespace
|
||||
return std::make_shared<Method>(
|
||||
argument_types.size() - 1,
|
||||
gradient_computer,
|
||||
weights_updater,
|
||||
weights_updater_name,
|
||||
learning_rate,
|
||||
l2_reg_coef,
|
||||
batch_size,
|
||||
argument_types,
|
||||
parameters);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionMLMethod(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("linearRegression", createAggregateFunctionMLMethod<FuncLinearRegression>);
|
||||
factory.registerFunction("logisticRegression", createAggregateFunctionMLMethod<FuncLogisticRegression>);
|
||||
factory.registerFunction("stochasticLinearRegression", createAggregateFunctionMLMethod<FuncLinearRegression>);
|
||||
factory.registerFunction("stochasticLogisticRegression", createAggregateFunctionMLMethod<FuncLogisticRegression>);
|
||||
}
|
||||
|
||||
LinearModelData::LinearModelData(
|
||||
@ -149,6 +139,26 @@ void LinearModelData::predict(
|
||||
gradient_computer->predict(container, block, arguments, weights, bias, context);
|
||||
}
|
||||
|
||||
void LinearModelData::returnWeights(IColumn & to) const
|
||||
{
|
||||
size_t size = weights.size() + 1;
|
||||
|
||||
ColumnArray & arr_to = static_cast<ColumnArray &>(to);
|
||||
ColumnArray::Offsets & offsets_to = arr_to.getOffsets();
|
||||
|
||||
size_t old_size = offsets_to.back();
|
||||
offsets_to.push_back(old_size + size);
|
||||
|
||||
typename ColumnFloat64::Container & val_to
|
||||
= static_cast<ColumnFloat64 &>(arr_to.getData()).getData();
|
||||
|
||||
val_to.reserve(old_size + size);
|
||||
for (size_t i = 0; i + 1 < size; ++i)
|
||||
val_to.push_back(weights[i]);
|
||||
|
||||
val_to.push_back(bias);
|
||||
}
|
||||
|
||||
void LinearModelData::read(ReadBuffer & buf)
|
||||
{
|
||||
readBinary(bias, buf);
|
||||
@ -192,7 +202,8 @@ void LinearModelData::merge(const DB::LinearModelData & rhs)
|
||||
void LinearModelData::add(const IColumn ** columns, size_t row_num)
|
||||
{
|
||||
/// first column stores target; features start from (columns + 1)
|
||||
const auto target = (*columns[0])[row_num].get<Float64>();
|
||||
Float64 target = (*columns[0]).getFloat64(row_num);
|
||||
|
||||
/// Here we have columns + 1 as first column corresponds to target value, and others - to features
|
||||
weights_updater->add_to_batch(
|
||||
gradient_batch, *gradient_computer, weights, bias, learning_rate, l2_reg_coef, target, columns + 1, row_num);
|
||||
@ -385,7 +396,7 @@ void LogisticRegression::compute(
|
||||
Float64 derivative = bias;
|
||||
for (size_t i = 0; i < weights.size(); ++i)
|
||||
{
|
||||
auto value = (*columns[i])[row_num].get<Float64>();
|
||||
auto value = (*columns[i]).getFloat64(row_num);
|
||||
derivative += weights[i] * value;
|
||||
}
|
||||
derivative *= target;
|
||||
@ -394,8 +405,8 @@ void LogisticRegression::compute(
|
||||
batch_gradient[weights.size()] += learning_rate * target / (derivative + 1);
|
||||
for (size_t i = 0; i < weights.size(); ++i)
|
||||
{
|
||||
auto value = (*columns[i])[row_num].get<Float64>();
|
||||
batch_gradient[i] += learning_rate * target * value / (derivative + 1) - 2 * l2_reg_coef * weights[i];
|
||||
auto value = (*columns[i]).getFloat64(row_num);
|
||||
batch_gradient[i] += learning_rate * target * value / (derivative + 1) - 2 * learning_rate * l2_reg_coef * weights[i];
|
||||
}
|
||||
}
|
||||
|
||||
@ -458,7 +469,7 @@ void LinearRegression::compute(
|
||||
Float64 derivative = (target - bias);
|
||||
for (size_t i = 0; i < weights.size(); ++i)
|
||||
{
|
||||
auto value = (*columns[i])[row_num].get<Float64>();
|
||||
auto value = (*columns[i]).getFloat64(row_num);
|
||||
derivative -= weights[i] * value;
|
||||
}
|
||||
derivative *= (2 * learning_rate);
|
||||
@ -466,8 +477,8 @@ void LinearRegression::compute(
|
||||
batch_gradient[weights.size()] += derivative;
|
||||
for (size_t i = 0; i < weights.size(); ++i)
|
||||
{
|
||||
auto value = (*columns[i])[row_num].get<Float64>();
|
||||
batch_gradient[i] += derivative * value - 2 * l2_reg_coef * weights[i];
|
||||
auto value = (*columns[i]).getFloat64(row_num);
|
||||
batch_gradient[i] += derivative * value - 2 * learning_rate * l2_reg_coef * weights[i];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -4,6 +4,8 @@
|
||||
#include <Columns/ColumnsCommon.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include "IAggregateFunction.h"
|
||||
|
||||
namespace DB
|
||||
@ -12,6 +14,7 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int BAD_CAST;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -218,6 +221,7 @@ public:
|
||||
void
|
||||
predict(ColumnVector<Float64>::Container & container, Block & block, const ColumnNumbers & arguments, const Context & context) const;
|
||||
|
||||
void returnWeights(IColumn & to) const;
|
||||
private:
|
||||
std::vector<Float64> weights;
|
||||
Float64 bias{0.0};
|
||||
@ -253,7 +257,7 @@ public:
|
||||
explicit AggregateFunctionMLMethod(
|
||||
UInt32 param_num,
|
||||
std::shared_ptr<IGradientComputer> gradient_computer,
|
||||
std::shared_ptr<IWeightsUpdater> weights_updater,
|
||||
std::string weights_updater_name,
|
||||
Float64 learning_rate,
|
||||
Float64 l2_reg_coef,
|
||||
UInt32 batch_size,
|
||||
@ -265,15 +269,39 @@ public:
|
||||
, l2_reg_coef(l2_reg_coef)
|
||||
, batch_size(batch_size)
|
||||
, gradient_computer(std::move(gradient_computer))
|
||||
, weights_updater(std::move(weights_updater))
|
||||
, weights_updater_name(std::move(weights_updater_name))
|
||||
{
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeNumber<Float64>>(); }
|
||||
/// This function is called when SELECT linearRegression(...) is called
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeFloat64>());
|
||||
}
|
||||
|
||||
/// This function is called from evalMLMethod function for correct predictValues call
|
||||
DataTypePtr getReturnTypeToPredict() const override
|
||||
{
|
||||
return std::make_shared<DataTypeNumber<Float64>>();
|
||||
}
|
||||
|
||||
void create(AggregateDataPtr place) const override
|
||||
{
|
||||
new (place) Data(learning_rate, l2_reg_coef, param_num, batch_size, gradient_computer, weights_updater);
|
||||
std::shared_ptr<IWeightsUpdater> new_weights_updater;
|
||||
if (weights_updater_name == "\'SGD\'")
|
||||
{
|
||||
new_weights_updater = std::make_shared<StochasticGradientDescent>();
|
||||
} else if (weights_updater_name == "\'Momentum\'")
|
||||
{
|
||||
new_weights_updater = std::make_shared<Momentum>();
|
||||
} else if (weights_updater_name == "\'Nesterov\'")
|
||||
{
|
||||
new_weights_updater = std::make_shared<Nesterov>();
|
||||
} else
|
||||
{
|
||||
throw Exception("Illegal name of weights updater (should have been checked earlier)", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
new (place) Data(learning_rate, l2_reg_coef, param_num, batch_size, gradient_computer, new_weights_updater);
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
@ -296,16 +324,26 @@ public:
|
||||
+ ". Required: " + std::to_string(param_num + 1),
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
auto & column = dynamic_cast<ColumnVector<Float64> &>(to);
|
||||
/// This cast might be correct because column type is based on getReturnTypeToPredict.
|
||||
ColumnVector<Float64> * column;
|
||||
try
|
||||
{
|
||||
column = &dynamic_cast<ColumnVector<Float64> &>(to);
|
||||
} catch (const std::bad_cast &)
|
||||
{
|
||||
throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to",
|
||||
ErrorCodes::BAD_CAST);
|
||||
}
|
||||
|
||||
this->data(place).predict(column.getData(), block, arguments, context);
|
||||
this->data(place).predict(column->getData(), block, arguments, context);
|
||||
}
|
||||
|
||||
/** This function is called if aggregate function without State modifier is selected in a query.
|
||||
* Inserts all weights of the model into the column 'to', so user may use such information if needed
|
||||
*/
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
std::ignore = place;
|
||||
std::ignore = to;
|
||||
throw std::runtime_error("not implemented");
|
||||
this->data(place).returnWeights(to);
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override { return __FILE__; }
|
||||
@ -316,15 +354,15 @@ private:
|
||||
Float64 l2_reg_coef;
|
||||
UInt32 batch_size;
|
||||
std::shared_ptr<IGradientComputer> gradient_computer;
|
||||
std::shared_ptr<IWeightsUpdater> weights_updater;
|
||||
std::string weights_updater_name;
|
||||
};
|
||||
|
||||
struct NameLinearRegression
|
||||
{
|
||||
static constexpr auto name = "linearRegression";
|
||||
static constexpr auto name = "stochasticLinearRegression";
|
||||
};
|
||||
struct NameLogisticRegression
|
||||
{
|
||||
static constexpr auto name = "logisticRegression";
|
||||
static constexpr auto name = "stochasticLogisticRegression";
|
||||
};
|
||||
}
|
||||
|
||||
@ -48,6 +48,12 @@ public:
|
||||
/// Get the result type.
|
||||
virtual DataTypePtr getReturnType() const = 0;
|
||||
|
||||
/// Get type which will be used for prediction result in case if function is an ML method.
|
||||
virtual DataTypePtr getReturnTypeToPredict() const
|
||||
{
|
||||
throw Exception("Prediction is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
virtual ~IAggregateFunction() {}
|
||||
|
||||
/** Data manipulating functions. */
|
||||
|
||||
@ -35,25 +35,6 @@ void ColumnAggregateFunction::addArena(ArenaPtr arena_)
|
||||
arenas.push_back(arena_);
|
||||
}
|
||||
|
||||
/// This function is used in convertToValues() and predictValues()
|
||||
/// and is written here to avoid repetitions
|
||||
bool ColumnAggregateFunction::tryFinalizeAggregateFunction(MutableColumnPtr *res_) const
|
||||
{
|
||||
if (const AggregateFunctionState *function_state = typeid_cast<const AggregateFunctionState *>(func.get()))
|
||||
{
|
||||
auto res = createView();
|
||||
res->set(function_state->getNestedFunction());
|
||||
res->data.assign(data.begin(), data.end());
|
||||
*res_ = std::move(res);
|
||||
return true;
|
||||
}
|
||||
|
||||
MutableColumnPtr res = func->getReturnType()->createColumn();
|
||||
res->reserve(data.size());
|
||||
*res_ = std::move(res);
|
||||
return false;
|
||||
}
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::convertToValues() const
|
||||
{
|
||||
/** If the aggregate function returns an unfinalized/unfinished state,
|
||||
@ -86,17 +67,17 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues() const
|
||||
* AggregateFunction(quantileTiming(0.5), UInt64)
|
||||
* into UInt16 - already finished result of `quantileTiming`.
|
||||
*/
|
||||
|
||||
/** Convertion function is used in convertToValues and predictValues
|
||||
* in the similar part of both functions
|
||||
*/
|
||||
|
||||
MutableColumnPtr res;
|
||||
if (tryFinalizeAggregateFunction(&res))
|
||||
if (const AggregateFunctionState *function_state = typeid_cast<const AggregateFunctionState *>(func.get()))
|
||||
{
|
||||
auto res = createView();
|
||||
res->set(function_state->getNestedFunction());
|
||||
res->data.assign(data.begin(), data.end());
|
||||
return res;
|
||||
}
|
||||
|
||||
MutableColumnPtr res = func->getReturnType()->createColumn();
|
||||
res->reserve(data.size());
|
||||
|
||||
for (auto val : data)
|
||||
func->insertResultInto(val, *res);
|
||||
|
||||
@ -105,8 +86,8 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues() const
|
||||
|
||||
MutableColumnPtr ColumnAggregateFunction::predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const
|
||||
{
|
||||
MutableColumnPtr res;
|
||||
tryFinalizeAggregateFunction(&res);
|
||||
MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn();
|
||||
res->reserve(data.size());
|
||||
|
||||
auto ML_function = func.get();
|
||||
if (ML_function)
|
||||
|
||||
@ -94,7 +94,7 @@ ColumnPtr ColumnDecimal<T>::permute(const IColumn::Permutation & perm, size_t li
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res_data[i] = data[perm[i]];
|
||||
|
||||
return std::move(res);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -117,7 +117,7 @@ MutableColumnPtr ColumnDecimal<T>::cloneResized(size_t size) const
|
||||
}
|
||||
}
|
||||
|
||||
return std::move(res);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -169,7 +169,7 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
|
||||
++data_pos;
|
||||
}
|
||||
|
||||
return std::move(res);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -202,7 +202,7 @@ ColumnPtr ColumnDecimal<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
res_data.push_back(data[i]);
|
||||
}
|
||||
|
||||
return std::move(res);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
|
||||
@ -187,7 +187,7 @@ ColumnPtr ColumnDecimal<T>::indexImpl(const PaddedPODArray<Type> & indexes, size
|
||||
for (size_t i = 0; i < limit; ++i)
|
||||
res_data[i] = data[indexes[i]];
|
||||
|
||||
return std::move(res);
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -349,7 +349,7 @@ ColumnPtr ColumnLowCardinality::countKeys() const
|
||||
|
||||
auto counter = ColumnUInt64::create(dict_size, 0);
|
||||
idx.countKeys(counter->getData());
|
||||
return std::move(counter);
|
||||
return counter;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -215,6 +215,12 @@ UInt64 ColumnVector<T>::get64(size_t n) const
|
||||
return ext::bit_cast<UInt64>(data[n]);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Float64 ColumnVector<T>::getFloat64(size_t n) const
|
||||
{
|
||||
return static_cast<Float64>(data[n]);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
|
||||
{
|
||||
|
||||
@ -202,6 +202,8 @@ public:
|
||||
|
||||
UInt64 get64(size_t n) const override;
|
||||
|
||||
Float64 getFloat64(size_t n) const override;
|
||||
|
||||
UInt64 getUInt(size_t n) const override
|
||||
{
|
||||
return UInt64(data[n]);
|
||||
|
||||
@ -91,6 +91,13 @@ public:
|
||||
throw Exception("Method get64 is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// If column stores native numeric type, it returns n-th element casted to Float64
|
||||
/// Is used in regression methods to cast each features into uniform type
|
||||
virtual Float64 getFloat64(size_t /*n*/) const
|
||||
{
|
||||
throw Exception("Method getFloat64 is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/** If column is numeric, return value of n-th element, casted to UInt64.
|
||||
* For NULL values of Nullable column it is allowed to return arbitrary value.
|
||||
* Otherwise throw an exception.
|
||||
|
||||
@ -375,7 +375,7 @@ ColumnUInt64::MutablePtr ReverseIndex<IndexType, ColumnType>::calcHashes() const
|
||||
for (auto row : ext::range(0, size))
|
||||
hash->getElement(row) = getHash(column->getDataAt(row));
|
||||
|
||||
return std::move(hash);
|
||||
return hash;
|
||||
}
|
||||
|
||||
template <typename IndexType, typename ColumnType>
|
||||
|
||||
67
dbms/src/Columns/getLeastSuperColumn.cpp
Normal file
67
dbms/src/Columns/getLeastSuperColumn.cpp
Normal file
@ -0,0 +1,67 @@
|
||||
#include <Columns/getLeastSuperColumn.h>
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
static bool sameConstants(const IColumn & a, const IColumn & b)
|
||||
{
|
||||
return static_cast<const ColumnConst &>(a).getField() == static_cast<const ColumnConst &>(b).getField();
|
||||
}
|
||||
|
||||
ColumnWithTypeAndName getLeastSuperColumn(std::vector<const ColumnWithTypeAndName *> columns)
|
||||
{
|
||||
if (columns.empty())
|
||||
throw Exception("Logical error: no src columns for supercolumn", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
ColumnWithTypeAndName result = *columns[0];
|
||||
|
||||
/// Determine common type.
|
||||
|
||||
size_t num_const = 0;
|
||||
DataTypes types(columns.size());
|
||||
for (size_t i = 0; i < columns.size(); ++i)
|
||||
{
|
||||
types[i] = columns[i]->type;
|
||||
if (columns[i]->column->isColumnConst())
|
||||
++num_const;
|
||||
}
|
||||
|
||||
result.type = getLeastSupertype(types);
|
||||
|
||||
/// Create supertype column saving constness if possible.
|
||||
|
||||
bool save_constness = false;
|
||||
if (columns.size() == num_const)
|
||||
{
|
||||
save_constness = true;
|
||||
for (size_t i = 1; i < columns.size(); ++i)
|
||||
{
|
||||
const ColumnWithTypeAndName & first = *columns[0];
|
||||
const ColumnWithTypeAndName & other = *columns[i];
|
||||
|
||||
if (!sameConstants(*first.column, *other.column))
|
||||
{
|
||||
save_constness = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (save_constness)
|
||||
result.column = result.type->createColumnConst(0, static_cast<const ColumnConst &>(*columns[0]->column).getField());
|
||||
else
|
||||
result.column = result.type->createColumn();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
12
dbms/src/Columns/getLeastSuperColumn.h
Normal file
12
dbms/src/Columns/getLeastSuperColumn.h
Normal file
@ -0,0 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/ColumnWithTypeAndName.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// getLeastSupertype + related column changes
|
||||
ColumnWithTypeAndName getLeastSuperColumn(std::vector<const ColumnWithTypeAndName *> columns);
|
||||
|
||||
}
|
||||
@ -427,6 +427,9 @@ namespace ErrorCodes
|
||||
extern const int BAD_TTL_EXPRESSION = 450;
|
||||
extern const int BAD_TTL_FILE = 451;
|
||||
extern const int SETTING_CONSTRAINT_VIOLATION = 452;
|
||||
extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES = 453;
|
||||
extern const int OPENSSL_ERROR = 454;
|
||||
extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY = 455;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
@ -436,8 +439,6 @@ namespace ErrorCodes
|
||||
|
||||
extern const int CONDITIONAL_TREE_PARENT_NOT_FOUND = 2001;
|
||||
extern const int ILLEGAL_PROJECTION_MANIPULATOR = 2002;
|
||||
|
||||
extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES = 446;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
18
dbms/src/Common/OpenSSLHelpers.cpp
Normal file
18
dbms/src/Common/OpenSSLHelpers.cpp
Normal file
@ -0,0 +1,18 @@
|
||||
#include "OpenSSLHelpers.h"
|
||||
#include <ext/scope_guard.h>
|
||||
#include <openssl/err.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
String getOpenSSLErrors()
|
||||
{
|
||||
BIO * mem = BIO_new(BIO_s_mem());
|
||||
SCOPE_EXIT(BIO_free(mem));
|
||||
ERR_print_errors(mem);
|
||||
char * buf = nullptr;
|
||||
long size = BIO_get_mem_data(mem, &buf);
|
||||
return String(buf, size);
|
||||
}
|
||||
|
||||
}
|
||||
12
dbms/src/Common/OpenSSLHelpers.h
Normal file
12
dbms/src/Common/OpenSSLHelpers.h
Normal file
@ -0,0 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.
|
||||
String getOpenSSLErrors();
|
||||
|
||||
}
|
||||
@ -63,7 +63,7 @@ template <typename T> bool inline operator> (T a, const UInt128 b) { return UIn
|
||||
template <typename T> bool inline operator<= (T a, const UInt128 b) { return UInt128(a) <= b; }
|
||||
template <typename T> bool inline operator< (T a, const UInt128 b) { return UInt128(a) < b; }
|
||||
|
||||
template <> constexpr bool IsNumber<UInt128> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt128> = true;
|
||||
template <> struct TypeName<UInt128> { static const char * get() { return "UInt128"; } };
|
||||
template <> struct TypeId<UInt128> { static constexpr const TypeIndex value = TypeIndex::UInt128; };
|
||||
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#include <Common/localBackup.h>
|
||||
#include "localBackup.h"
|
||||
|
||||
#include <Common/createHardLink.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Poco/Path.h>
|
||||
#include <Poco/File.h>
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Path.h>
|
||||
#include <optional>
|
||||
|
||||
namespace Poco { class Path; }
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -1,18 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <Core/Types.h>
|
||||
#include <IO/copyData.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <IO/ReadBufferFromPocoSocket.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteBufferFromPocoSocket.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Poco/RandomStream.h>
|
||||
#include <Poco/Net/StreamSocket.h>
|
||||
#include <Poco/RandomStream.h>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Poco/Logger.h>
|
||||
|
||||
/// Implementation of MySQL wire protocol
|
||||
|
||||
@ -157,23 +155,19 @@ public:
|
||||
size_t max_packet_size = MAX_PACKET_LENGTH;
|
||||
|
||||
/// For reading and writing.
|
||||
PacketSender(ReadBuffer & in, WriteBuffer & out, size_t & sequence_id, const String logger_name)
|
||||
PacketSender(ReadBuffer & in, WriteBuffer & out, size_t & sequence_id)
|
||||
: sequence_id(sequence_id)
|
||||
, in(&in)
|
||||
, out(&out)
|
||||
, log(&Poco::Logger::get(logger_name))
|
||||
{
|
||||
log->setLevel("information");
|
||||
}
|
||||
|
||||
/// For writing.
|
||||
PacketSender(WriteBuffer & out, size_t & sequence_id, const String logger_name)
|
||||
PacketSender(WriteBuffer & out, size_t & sequence_id)
|
||||
: sequence_id(sequence_id)
|
||||
, in(nullptr)
|
||||
, out(&out)
|
||||
, log(&Poco::Logger::get(logger_name))
|
||||
{
|
||||
log->setLevel("information");
|
||||
}
|
||||
|
||||
String receivePacketPayload()
|
||||
@ -186,8 +180,6 @@ public:
|
||||
// packets which are larger than or equal to 16MB are splitted
|
||||
do
|
||||
{
|
||||
LOG_TRACE(log, "Reading from buffer");
|
||||
|
||||
in->readStrict(reinterpret_cast<char *>(&payload_length), 3);
|
||||
|
||||
if (payload_length > max_packet_size)
|
||||
@ -207,8 +199,6 @@ public:
|
||||
}
|
||||
sequence_id++;
|
||||
|
||||
LOG_TRACE(log, "Received packet. Sequence-id: " << packet_sequence_id << ", payload length: " << payload_length);
|
||||
|
||||
copyData(*in, static_cast<WriteBuffer &>(buf), payload_length);
|
||||
} while (payload_length == max_packet_size);
|
||||
|
||||
@ -230,9 +220,6 @@ public:
|
||||
{
|
||||
size_t payload_length = std::min(payload.length() - pos, max_packet_size);
|
||||
|
||||
LOG_TRACE(log, "Writing packet of size " << payload_length << " with sequence-id " << static_cast<int>(sequence_id));
|
||||
LOG_TRACE(log, packetToText(payload));
|
||||
|
||||
out->write(reinterpret_cast<const char *>(&payload_length), 3);
|
||||
out->write(reinterpret_cast<const char *>(&sequence_id), 1);
|
||||
out->write(payload.data() + pos, payload_length);
|
||||
@ -241,12 +228,8 @@ public:
|
||||
sequence_id++;
|
||||
} while (pos < payload.length());
|
||||
|
||||
LOG_TRACE(log, "Packet was sent.");
|
||||
|
||||
if (flush)
|
||||
{
|
||||
out->next();
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets sequence-id to 0. Must be called before each command phase.
|
||||
@ -255,8 +238,6 @@ public:
|
||||
private:
|
||||
/// Converts packet to text. Is used for debug output.
|
||||
static String packetToText(String payload);
|
||||
|
||||
Poco::Logger * log;
|
||||
};
|
||||
|
||||
|
||||
@ -279,12 +260,11 @@ class Handshake : public WritePacket
|
||||
uint32_t status_flags;
|
||||
String auth_plugin_data;
|
||||
public:
|
||||
explicit Handshake(uint32_t connection_id, String server_version, String auth_plugin_data)
|
||||
explicit Handshake(uint32_t capability_flags, uint32_t connection_id, String server_version, String auth_plugin_data)
|
||||
: protocol_version(0xa)
|
||||
, server_version(std::move(server_version))
|
||||
, connection_id(connection_id)
|
||||
, capability_flags(CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA
|
||||
| CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF | CLIENT_SSL)
|
||||
, capability_flags(capability_flags)
|
||||
, character_set(CharacterSet::utf8_general_ci)
|
||||
, status_flags(0)
|
||||
, auth_plugin_data(auth_plugin_data)
|
||||
|
||||
@ -85,6 +85,7 @@ struct Settings : public SettingsCollection<Settings>
|
||||
M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
|
||||
\
|
||||
M(SettingBool, compile, false, "Whether query compilation is enabled.") \
|
||||
M(SettingBool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.") \
|
||||
M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \
|
||||
M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
|
||||
M(SettingUInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled") \
|
||||
|
||||
@ -32,16 +32,16 @@ using String = std::string;
|
||||
*/
|
||||
template <typename T> constexpr bool IsNumber = false;
|
||||
|
||||
template <> constexpr bool IsNumber<UInt8> = true;
|
||||
template <> constexpr bool IsNumber<UInt16> = true;
|
||||
template <> constexpr bool IsNumber<UInt32> = true;
|
||||
template <> constexpr bool IsNumber<UInt64> = true;
|
||||
template <> constexpr bool IsNumber<Int8> = true;
|
||||
template <> constexpr bool IsNumber<Int16> = true;
|
||||
template <> constexpr bool IsNumber<Int32> = true;
|
||||
template <> constexpr bool IsNumber<Int64> = true;
|
||||
template <> constexpr bool IsNumber<Float32> = true;
|
||||
template <> constexpr bool IsNumber<Float64> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt8> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt16> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt32> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt64> = true;
|
||||
template <> inline constexpr bool IsNumber<Int8> = true;
|
||||
template <> inline constexpr bool IsNumber<Int16> = true;
|
||||
template <> inline constexpr bool IsNumber<Int32> = true;
|
||||
template <> inline constexpr bool IsNumber<Int64> = true;
|
||||
template <> inline constexpr bool IsNumber<Float32> = true;
|
||||
template <> inline constexpr bool IsNumber<Float64> = true;
|
||||
|
||||
template <typename T> struct TypeName;
|
||||
|
||||
@ -109,8 +109,8 @@ using Strings = std::vector<String>;
|
||||
|
||||
|
||||
using Int128 = __int128;
|
||||
template <> constexpr bool IsNumber<Int128> = true;
|
||||
template <> struct TypeName<Int128> { static const char * get() { return "Int128"; } };
|
||||
template <> inline constexpr bool IsNumber<Int128> = true;
|
||||
template <> struct TypeName<Int128> { static const char * get() { return "Int128"; } };
|
||||
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
|
||||
|
||||
/// Own FieldType for Decimal.
|
||||
@ -159,17 +159,56 @@ template <> struct TypeId<Decimal32> { static constexpr const TypeIndex value
|
||||
template <> struct TypeId<Decimal64> { static constexpr const TypeIndex value = TypeIndex::Decimal64; };
|
||||
template <> struct TypeId<Decimal128> { static constexpr const TypeIndex value = TypeIndex::Decimal128; };
|
||||
|
||||
template <typename T>
|
||||
constexpr bool IsDecimalNumber = false;
|
||||
template <> constexpr bool IsDecimalNumber<Decimal32> = true;
|
||||
template <> constexpr bool IsDecimalNumber<Decimal64> = true;
|
||||
template <> constexpr bool IsDecimalNumber<Decimal128> = true;
|
||||
template <typename T> constexpr bool IsDecimalNumber = false;
|
||||
template <> inline constexpr bool IsDecimalNumber<Decimal32> = true;
|
||||
template <> inline constexpr bool IsDecimalNumber<Decimal64> = true;
|
||||
template <> inline constexpr bool IsDecimalNumber<Decimal128> = true;
|
||||
|
||||
template <typename T> struct NativeType { using Type = T; };
|
||||
template <> struct NativeType<Decimal32> { using Type = Int32; };
|
||||
template <> struct NativeType<Decimal64> { using Type = Int64; };
|
||||
template <> struct NativeType<Decimal128> { using Type = Int128; };
|
||||
|
||||
inline const char * getTypeName(TypeIndex idx)
|
||||
{
|
||||
switch (idx)
|
||||
{
|
||||
case TypeIndex::Nothing: return "Nothing";
|
||||
case TypeIndex::UInt8: return TypeName<UInt8>::get();
|
||||
case TypeIndex::UInt16: return TypeName<UInt16>::get();
|
||||
case TypeIndex::UInt32: return TypeName<UInt32>::get();
|
||||
case TypeIndex::UInt64: return TypeName<UInt64>::get();
|
||||
case TypeIndex::UInt128: return "UInt128";
|
||||
case TypeIndex::Int8: return TypeName<Int8>::get();
|
||||
case TypeIndex::Int16: return TypeName<Int16>::get();
|
||||
case TypeIndex::Int32: return TypeName<Int32>::get();
|
||||
case TypeIndex::Int64: return TypeName<Int64>::get();
|
||||
case TypeIndex::Int128: return TypeName<Int128>::get();
|
||||
case TypeIndex::Float32: return TypeName<Float32>::get();
|
||||
case TypeIndex::Float64: return TypeName<Float64>::get();
|
||||
case TypeIndex::Date: return "Date";
|
||||
case TypeIndex::DateTime: return "DateTime";
|
||||
case TypeIndex::String: return TypeName<String>::get();
|
||||
case TypeIndex::FixedString: return "FixedString";
|
||||
case TypeIndex::Enum8: return "Enum8";
|
||||
case TypeIndex::Enum16: return "Enum16";
|
||||
case TypeIndex::Decimal32: return TypeName<Decimal32>::get();
|
||||
case TypeIndex::Decimal64: return TypeName<Decimal64>::get();
|
||||
case TypeIndex::Decimal128: return TypeName<Decimal128>::get();
|
||||
case TypeIndex::UUID: return "UUID";
|
||||
case TypeIndex::Array: return "Array";
|
||||
case TypeIndex::Tuple: return "Tuple";
|
||||
case TypeIndex::Set: return "Set";
|
||||
case TypeIndex::Interval: return "Interval";
|
||||
case TypeIndex::Nullable: return "Nullable";
|
||||
case TypeIndex::Function: return "Function";
|
||||
case TypeIndex::AggregateFunction: return "AggregateFunction";
|
||||
case TypeIndex::LowCardinality: return "LowCardinality";
|
||||
}
|
||||
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Specialization of `std::hash` for the Decimal<T> types.
|
||||
|
||||
@ -36,6 +36,7 @@ public:
|
||||
bool canBeInsideNullable() const override { return false; }
|
||||
|
||||
DataTypePtr getReturnType() const { return function->getReturnType(); }
|
||||
DataTypePtr getReturnTypeToPredict() const { return function->getReturnTypeToPredict(); }
|
||||
DataTypes getArgumentsDataTypes() const { return argument_types; }
|
||||
|
||||
/// NOTE These two functions for serializing single values are incompatible with the functions below.
|
||||
|
||||
@ -111,7 +111,7 @@ ColumnPtr recursiveLowCardinalityConversion(const ColumnPtr & column, const Data
|
||||
{
|
||||
auto col = low_cardinality_type->createColumn();
|
||||
static_cast<ColumnLowCardinality &>(*col).insertRangeFromFullColumn(*column, 0, column->size());
|
||||
return std::move(col);
|
||||
return col;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -246,9 +246,9 @@ inline UInt32 getDecimalScale(const IDataType & data_type, UInt32 default_value
|
||||
///
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimal = false;
|
||||
template <> constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal32>> = true;
|
||||
template <> constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal64>> = true;
|
||||
template <> constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128>> = true;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128>> = true;
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeDecimalOrNumber = IsDataTypeDecimal<DataType> || IsDataTypeNumber<DataType>;
|
||||
|
||||
|
||||
@ -38,15 +38,15 @@ using DataTypeFloat32 = DataTypeNumber<Float32>;
|
||||
using DataTypeFloat64 = DataTypeNumber<Float64>;
|
||||
|
||||
template <typename DataType> constexpr bool IsDataTypeNumber = false;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<UInt8>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<UInt16>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<UInt32>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<UInt64>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Int8>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Int16>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Int32>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Int64>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Float32>> = true;
|
||||
template <> constexpr bool IsDataTypeNumber<DataTypeNumber<Float64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt8>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt16>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<UInt64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int8>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int16>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Int64>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Float32>> = true;
|
||||
template <> inline constexpr bool IsDataTypeNumber<DataTypeNumber<Float64>> = true;
|
||||
|
||||
}
|
||||
|
||||
@ -86,7 +86,7 @@ void CacheDictionary::toParent(const PaddedPODArray<Key> & ids, PaddedPODArray<K
|
||||
{
|
||||
const auto null_value = std::get<UInt64>(hierarchical_attribute->null_values);
|
||||
|
||||
getItemsNumber<UInt64>(*hierarchical_attribute, ids, out, [&](const size_t) { return null_value; });
|
||||
getItemsNumberImpl<UInt64, UInt64>(*hierarchical_attribute, ids, out, [&](const size_t) { return null_value; });
|
||||
}
|
||||
|
||||
|
||||
@ -207,9 +207,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
|
||||
void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||
|
||||
@ -220,9 +218,7 @@ void CacheDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsString(attribute, ids, out, [&](const size_t row) { return def->getDataAt(row); });
|
||||
}
|
||||
@ -231,9 +227,7 @@ void CacheDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsString(attribute, ids, out, [&](const size_t) { return StringRef{def}; });
|
||||
}
|
||||
|
||||
@ -221,11 +221,6 @@ private:
|
||||
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
|
||||
template <typename OutputType, typename DefaultGetter>
|
||||
void getItemsNumber(
|
||||
Attribute & attribute, const PaddedPODArray<Key> & ids, ResultArrayType<OutputType> & out, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename DefaultGetter>
|
||||
void getItemsNumberImpl(
|
||||
Attribute & attribute, const PaddedPODArray<Key> & ids, ResultArrayType<OutputType> & out, DefaultGetter && get_default) const;
|
||||
|
||||
@ -34,34 +34,6 @@ namespace ErrorCodes
|
||||
extern const int TYPE_MISMATCH;
|
||||
}
|
||||
|
||||
template <typename OutputType, typename DefaultGetter>
|
||||
void CacheDictionary::getItemsNumber(
|
||||
Attribute & attribute, const PaddedPODArray<Key> & ids, ResultArrayType<OutputType> & out, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) \
|
||||
getItemsNumberImpl<TYPE, OutputType>(attribute, ids, out, std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename DefaultGetter>
|
||||
void CacheDictionary::getItemsNumberImpl(
|
||||
Attribute & attribute, const PaddedPODArray<Key> & ids, ResultArrayType<OutputType> & out, DefaultGetter && get_default) const
|
||||
|
||||
@ -12,13 +12,11 @@ using TYPE = @NAME@;
|
||||
void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return null_value; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, ids, out, [&](const size_t) { return null_value; });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -15,11 +15,9 @@ void CacheDictionary::get@NAME@(const std::string & attribute_name,
|
||||
ResultArrayType<TYPE> & out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t row) { return def[row]; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, ids, out, [&](const size_t row) { return def[row]; });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -12,11 +12,9 @@ using TYPE = @NAME@;
|
||||
void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const
|
||||
{
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return def; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, ids, out, [&](const size_t) { return def; });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -77,9 +77,7 @@ void ComplexKeyCacheDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||
|
||||
@ -96,9 +94,7 @@ void ComplexKeyCacheDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsString(attribute, key_columns, out, [&](const size_t row) { return def->getDataAt(row); });
|
||||
}
|
||||
@ -113,9 +109,7 @@ void ComplexKeyCacheDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsString(attribute, key_columns, out, [&](const size_t) { return StringRef{def}; });
|
||||
}
|
||||
|
||||
@ -256,34 +256,6 @@ private:
|
||||
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
template <typename OutputType, typename DefaultGetter>
|
||||
void
|
||||
getItemsNumber(Attribute & attribute, const Columns & key_columns, PaddedPODArray<OutputType> & out, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) \
|
||||
getItemsNumberImpl<TYPE, OutputType>(attribute, key_columns, out, std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename DefaultGetter>
|
||||
void getItemsNumberImpl(
|
||||
Attribute & attribute, const Columns & key_columns, PaddedPODArray<OutputType> & out, DefaultGetter && get_default) const
|
||||
|
||||
@ -13,12 +13,10 @@ void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, co
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,10 +18,8 @@ void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,10 +18,8 @@ void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
|
||||
throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::@NAME@);
|
||||
|
||||
getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });
|
||||
getItemsNumberImpl<TYPE, TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });
|
||||
}
|
||||
}
|
||||
|
||||
@ -50,13 +50,11 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(
|
||||
dict_struct.validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, \
|
||||
key_columns, \
|
||||
[&](const size_t row, const auto value) { out[row] = value; }, \
|
||||
@ -84,9 +82,7 @@ void ComplexKeyHashedDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||
|
||||
@ -108,11 +104,9 @@ void ComplexKeyHashedDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, \
|
||||
key_columns, \
|
||||
[&](const size_t row, const auto value) { out[row] = value; }, \
|
||||
@ -144,9 +138,7 @@ void ComplexKeyHashedDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -166,11 +158,9 @@ void ComplexKeyHashedDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -199,9 +189,7 @@ void ComplexKeyHashedDictionary::getString(
|
||||
dict_struct.validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -566,34 +554,6 @@ ComplexKeyHashedDictionary::createAttributeWithType(const AttributeUnderlyingTyp
|
||||
}
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void ComplexKeyHashedDictionary::getItemsNumber(
|
||||
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) getItemsImpl<TYPE, OutputType>( \
|
||||
attribute, key_columns, std::forward<ValueSetter>(set_value), std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void ComplexKeyHashedDictionary::getItemsImpl(
|
||||
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
|
||||
@ -218,16 +218,10 @@ private:
|
||||
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void
|
||||
getItemsNumber(const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void
|
||||
getItemsImpl(const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
|
||||
template <typename T>
|
||||
bool setAttributeValueImpl(Attribute & attribute, const StringRef key, const T value);
|
||||
|
||||
|
||||
@ -40,44 +40,6 @@ namespace
|
||||
} // namespace
|
||||
|
||||
|
||||
bool isAttributeTypeConvertibleTo(AttributeUnderlyingType from, AttributeUnderlyingType to)
|
||||
{
|
||||
if (from == to)
|
||||
return true;
|
||||
|
||||
/** This enum can be somewhat incomplete and the meaning may not coincide with NumberTraits.h.
|
||||
* (for example, because integers can not be converted to floats)
|
||||
* This is normal for a limited usage scope.
|
||||
*/
|
||||
if ((from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::UInt16)
|
||||
|| (from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::UInt32)
|
||||
|| (from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::UInt64)
|
||||
|| (from == AttributeUnderlyingType::UInt16 && to == AttributeUnderlyingType::UInt32)
|
||||
|| (from == AttributeUnderlyingType::UInt16 && to == AttributeUnderlyingType::UInt64)
|
||||
|| (from == AttributeUnderlyingType::UInt32 && to == AttributeUnderlyingType::UInt64)
|
||||
|| (from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::Int16)
|
||||
|| (from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::Int32)
|
||||
|| (from == AttributeUnderlyingType::UInt8 && to == AttributeUnderlyingType::Int64)
|
||||
|| (from == AttributeUnderlyingType::UInt16 && to == AttributeUnderlyingType::Int32)
|
||||
|| (from == AttributeUnderlyingType::UInt16 && to == AttributeUnderlyingType::Int64)
|
||||
|| (from == AttributeUnderlyingType::UInt32 && to == AttributeUnderlyingType::Int64)
|
||||
|
||||
|| (from == AttributeUnderlyingType::Int8 && to == AttributeUnderlyingType::Int16)
|
||||
|| (from == AttributeUnderlyingType::Int8 && to == AttributeUnderlyingType::Int32)
|
||||
|| (from == AttributeUnderlyingType::Int8 && to == AttributeUnderlyingType::Int64)
|
||||
|| (from == AttributeUnderlyingType::Int16 && to == AttributeUnderlyingType::Int32)
|
||||
|| (from == AttributeUnderlyingType::Int16 && to == AttributeUnderlyingType::Int64)
|
||||
|| (from == AttributeUnderlyingType::Int32 && to == AttributeUnderlyingType::Int64)
|
||||
|
||||
|| (from == AttributeUnderlyingType::Float32 && to == AttributeUnderlyingType::Float64))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
AttributeUnderlyingType getAttributeUnderlyingType(const std::string & type)
|
||||
{
|
||||
static const std::unordered_map<std::string, AttributeUnderlyingType> dictionary{
|
||||
|
||||
@ -13,6 +13,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TYPE_MISMATCH;
|
||||
}
|
||||
|
||||
enum class AttributeUnderlyingType
|
||||
{
|
||||
UInt8,
|
||||
@ -33,14 +39,18 @@ enum class AttributeUnderlyingType
|
||||
};
|
||||
|
||||
|
||||
/** For implicit conversions in dictGet functions.
|
||||
*/
|
||||
bool isAttributeTypeConvertibleTo(AttributeUnderlyingType from, AttributeUnderlyingType to);
|
||||
|
||||
AttributeUnderlyingType getAttributeUnderlyingType(const std::string & type);
|
||||
|
||||
std::string toString(const AttributeUnderlyingType type);
|
||||
|
||||
/// Implicit conversions in dictGet functions is disabled.
|
||||
inline void checkAttributeType(const std::string & dict_name, const std::string & attribute_name,
|
||||
AttributeUnderlyingType attribute_type, AttributeUnderlyingType to)
|
||||
{
|
||||
if (attribute_type != to)
|
||||
throw Exception{dict_name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute_type)
|
||||
+ ", expected " + toString(to), ErrorCodes::TYPE_MISMATCH};
|
||||
}
|
||||
|
||||
/// Min and max lifetimes for a dictionary or it's entry
|
||||
using DictionaryLifetime = ExternalLoadableLifetime;
|
||||
|
||||
@ -55,7 +55,7 @@ void FlatDictionary::toParent(const PaddedPODArray<Key> & ids, PaddedPODArray<Ke
|
||||
{
|
||||
const auto null_value = std::get<UInt64>(hierarchical_attribute->null_values);
|
||||
|
||||
getItemsNumber<UInt64>(
|
||||
getItemsImpl<UInt64, UInt64>(
|
||||
*hierarchical_attribute,
|
||||
ids,
|
||||
[&](const size_t row, const UInt64 value) { out[row] = value; },
|
||||
@ -117,13 +117,11 @@ void FlatDictionary::isInConstantVector(const Key child_id, const PaddedPODArray
|
||||
void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return null_value; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -145,9 +143,7 @@ DECLARE(Decimal128)
|
||||
void FlatDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto & null_value = std::get<StringRef>(attribute.null_values);
|
||||
|
||||
@ -166,11 +162,9 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
|
||||
ResultArrayType<TYPE> & out) const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -193,9 +187,7 @@ void FlatDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -209,11 +201,9 @@ void FlatDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -236,9 +226,7 @@ void FlatDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
FlatDictionary::getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -580,35 +568,6 @@ FlatDictionary::Attribute FlatDictionary::createAttributeWithType(const Attribut
|
||||
}
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void FlatDictionary::getItemsNumber(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) \
|
||||
getItemsImpl<TYPE, OutputType>(attribute, ids, std::forward<ValueSetter>(set_value), std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void FlatDictionary::getItemsImpl(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
@ -701,10 +660,10 @@ void FlatDictionary::setAttributeValue(Attribute & attribute, const Key id, cons
|
||||
break;
|
||||
|
||||
case AttributeUnderlyingType::Decimal32:
|
||||
setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal128>());
|
||||
setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal32>());
|
||||
break;
|
||||
case AttributeUnderlyingType::Decimal64:
|
||||
setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal128>());
|
||||
setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal64>());
|
||||
break;
|
||||
case AttributeUnderlyingType::Decimal128:
|
||||
setAttributeValueImpl<Decimal128>(attribute, id, value.get<Decimal128>());
|
||||
|
||||
@ -206,10 +206,6 @@ private:
|
||||
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void getItemsNumber(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void getItemsImpl(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
@ -49,7 +49,7 @@ void HashedDictionary::toParent(const PaddedPODArray<Key> & ids, PaddedPODArray<
|
||||
{
|
||||
const auto null_value = std::get<UInt64>(hierarchical_attribute->null_values);
|
||||
|
||||
getItemsNumber<UInt64>(
|
||||
getItemsImpl<UInt64, UInt64>(
|
||||
*hierarchical_attribute,
|
||||
ids,
|
||||
[&](const size_t row, const UInt64 value) { out[row] = value; },
|
||||
@ -116,13 +116,11 @@ void HashedDictionary::isInConstantVector(const Key child_id, const PaddedPODArr
|
||||
const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return null_value; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -144,9 +142,7 @@ DECLARE(Decimal128)
|
||||
void HashedDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||
|
||||
@ -165,11 +161,9 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
|
||||
ResultArrayType<TYPE> & out) const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row) { return def[row]; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -192,9 +186,7 @@ void HashedDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def, ColumnString * const out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -208,11 +200,9 @@ void HashedDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE & def, ResultArrayType<TYPE> & out) const \
|
||||
{ \
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, ids, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -235,9 +225,7 @@ void HashedDictionary::getString(
|
||||
const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def, ColumnString * const out) const
|
||||
{
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -324,7 +312,6 @@ void HashedDictionary::createAttributes()
|
||||
void HashedDictionary::blockToAttributes(const Block & block)
|
||||
{
|
||||
const auto & id_column = *block.safeGetByPosition(0).column;
|
||||
element_count += id_column.size();
|
||||
|
||||
for (const size_t attribute_idx : ext::range(0, attributes.size()))
|
||||
{
|
||||
@ -332,7 +319,8 @@ void HashedDictionary::blockToAttributes(const Block & block)
|
||||
auto & attribute = attributes[attribute_idx];
|
||||
|
||||
for (const auto row_idx : ext::range(0, id_column.size()))
|
||||
setAttributeValue(attribute, id_column[row_idx].get<UInt64>(), attribute_column[row_idx]);
|
||||
if (setAttributeValue(attribute, id_column[row_idx].get<UInt64>(), attribute_column[row_idx]))
|
||||
++element_count;
|
||||
}
|
||||
}
|
||||
|
||||
@ -567,34 +555,6 @@ HashedDictionary::Attribute HashedDictionary::createAttributeWithType(const Attr
|
||||
}
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void HashedDictionary::getItemsNumber(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) \
|
||||
getItemsImpl<TYPE, OutputType>(attribute, ids, std::forward<ValueSetter>(set_value), std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void HashedDictionary::getItemsImpl(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
@ -613,69 +573,56 @@ void HashedDictionary::getItemsImpl(
|
||||
|
||||
|
||||
template <typename T>
|
||||
void HashedDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, const T value)
|
||||
bool HashedDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, const T value)
|
||||
{
|
||||
auto & map = *std::get<CollectionPtrType<T>>(attribute.maps);
|
||||
map.insert({id, value});
|
||||
return map.insert({id, value}).second;
|
||||
}
|
||||
|
||||
void HashedDictionary::setAttributeValue(Attribute & attribute, const Key id, const Field & value)
|
||||
bool HashedDictionary::setAttributeValue(Attribute & attribute, const Key id, const Field & value)
|
||||
{
|
||||
switch (attribute.type)
|
||||
{
|
||||
case AttributeUnderlyingType::UInt8:
|
||||
setAttributeValueImpl<UInt8>(attribute, id, value.get<UInt64>());
|
||||
break;
|
||||
return setAttributeValueImpl<UInt8>(attribute, id, value.get<UInt64>());
|
||||
case AttributeUnderlyingType::UInt16:
|
||||
setAttributeValueImpl<UInt16>(attribute, id, value.get<UInt64>());
|
||||
break;
|
||||
return setAttributeValueImpl<UInt16>(attribute, id, value.get<UInt64>());
|
||||
case AttributeUnderlyingType::UInt32:
|
||||
setAttributeValueImpl<UInt32>(attribute, id, value.get<UInt64>());
|
||||
break;
|
||||
return setAttributeValueImpl<UInt32>(attribute, id, value.get<UInt64>());
|
||||
case AttributeUnderlyingType::UInt64:
|
||||
setAttributeValueImpl<UInt64>(attribute, id, value.get<UInt64>());
|
||||
break;
|
||||
return setAttributeValueImpl<UInt64>(attribute, id, value.get<UInt64>());
|
||||
case AttributeUnderlyingType::UInt128:
|
||||
setAttributeValueImpl<UInt128>(attribute, id, value.get<UInt128>());
|
||||
break;
|
||||
return setAttributeValueImpl<UInt128>(attribute, id, value.get<UInt128>());
|
||||
case AttributeUnderlyingType::Int8:
|
||||
setAttributeValueImpl<Int8>(attribute, id, value.get<Int64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Int8>(attribute, id, value.get<Int64>());
|
||||
case AttributeUnderlyingType::Int16:
|
||||
setAttributeValueImpl<Int16>(attribute, id, value.get<Int64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Int16>(attribute, id, value.get<Int64>());
|
||||
case AttributeUnderlyingType::Int32:
|
||||
setAttributeValueImpl<Int32>(attribute, id, value.get<Int64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Int32>(attribute, id, value.get<Int64>());
|
||||
case AttributeUnderlyingType::Int64:
|
||||
setAttributeValueImpl<Int64>(attribute, id, value.get<Int64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Int64>(attribute, id, value.get<Int64>());
|
||||
case AttributeUnderlyingType::Float32:
|
||||
setAttributeValueImpl<Float32>(attribute, id, value.get<Float64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Float32>(attribute, id, value.get<Float64>());
|
||||
case AttributeUnderlyingType::Float64:
|
||||
setAttributeValueImpl<Float64>(attribute, id, value.get<Float64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Float64>(attribute, id, value.get<Float64>());
|
||||
|
||||
case AttributeUnderlyingType::Decimal32:
|
||||
setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal32>());
|
||||
break;
|
||||
return setAttributeValueImpl<Decimal32>(attribute, id, value.get<Decimal32>());
|
||||
case AttributeUnderlyingType::Decimal64:
|
||||
setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal64>());
|
||||
break;
|
||||
return setAttributeValueImpl<Decimal64>(attribute, id, value.get<Decimal64>());
|
||||
case AttributeUnderlyingType::Decimal128:
|
||||
setAttributeValueImpl<Decimal128>(attribute, id, value.get<Decimal128>());
|
||||
break;
|
||||
return setAttributeValueImpl<Decimal128>(attribute, id, value.get<Decimal128>());
|
||||
|
||||
case AttributeUnderlyingType::String:
|
||||
{
|
||||
auto & map = *std::get<CollectionPtrType<StringRef>>(attribute.maps);
|
||||
const auto & string = value.get<String>();
|
||||
const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size());
|
||||
map.insert({id, StringRef{string_in_arena, string.size()}});
|
||||
break;
|
||||
return map.insert({id, StringRef{string_in_arena, string.size()}}).second;
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception{"Invalid attribute type", ErrorCodes::BAD_ARGUMENTS};
|
||||
}
|
||||
|
||||
const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::string & attribute_name) const
|
||||
|
||||
@ -211,18 +211,14 @@ private:
|
||||
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void getItemsNumber(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void getItemsImpl(
|
||||
const Attribute & attribute, const PaddedPODArray<Key> & ids, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename T>
|
||||
void setAttributeValueImpl(Attribute & attribute, const Key id, const T value);
|
||||
bool setAttributeValueImpl(Attribute & attribute, const Key id, const T value);
|
||||
|
||||
void setAttributeValue(Attribute & attribute, const Key id, const Field & value);
|
||||
bool setAttributeValue(Attribute & attribute, const Key id, const Field & value);
|
||||
|
||||
const Attribute & getAttribute(const std::string & attribute_name) const;
|
||||
|
||||
|
||||
@ -75,13 +75,11 @@ TrieDictionary::~TrieDictionary()
|
||||
validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
const auto null_value = std::get<TYPE>(attribute.null_values); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, \
|
||||
key_columns, \
|
||||
[&](const size_t row, const auto value) { out[row] = value; }, \
|
||||
@ -109,9 +107,7 @@ void TrieDictionary::getString(
|
||||
validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
|
||||
|
||||
@ -133,11 +129,9 @@ void TrieDictionary::getString(
|
||||
validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, \
|
||||
key_columns, \
|
||||
[&](const size_t row, const auto value) { out[row] = value; }, \
|
||||
@ -169,9 +163,7 @@ void TrieDictionary::getString(
|
||||
validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -191,11 +183,9 @@ void TrieDictionary::getString(
|
||||
validateKeyTypes(key_types); \
|
||||
\
|
||||
const auto & attribute = getAttribute(attribute_name); \
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE)) \
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
|
||||
ErrorCodes::TYPE_MISMATCH}; \
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::TYPE); \
|
||||
\
|
||||
getItemsNumber<TYPE>( \
|
||||
getItemsImpl<TYPE, TYPE>( \
|
||||
attribute, key_columns, [&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t) { return def; }); \
|
||||
}
|
||||
DECLARE(UInt8)
|
||||
@ -224,9 +214,7 @@ void TrieDictionary::getString(
|
||||
validateKeyTypes(key_types);
|
||||
|
||||
const auto & attribute = getAttribute(attribute_name);
|
||||
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
|
||||
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
checkAttributeType(name, attribute_name, attribute.type, AttributeUnderlyingType::String);
|
||||
|
||||
getItemsImpl<StringRef, StringRef>(
|
||||
attribute,
|
||||
@ -507,34 +495,6 @@ TrieDictionary::Attribute TrieDictionary::createAttributeWithType(const Attribut
|
||||
}
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void TrieDictionary::getItemsNumber(
|
||||
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
{
|
||||
if (false)
|
||||
{
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (attribute.type == AttributeUnderlyingType::TYPE) getItemsImpl<TYPE, OutputType>( \
|
||||
attribute, key_columns, std::forward<ValueSetter>(set_value), std::forward<DefaultGetter>(get_default));
|
||||
DISPATCH(UInt8)
|
||||
DISPATCH(UInt16)
|
||||
DISPATCH(UInt32)
|
||||
DISPATCH(UInt64)
|
||||
DISPATCH(UInt128)
|
||||
DISPATCH(Int8)
|
||||
DISPATCH(Int16)
|
||||
DISPATCH(Int32)
|
||||
DISPATCH(Int64)
|
||||
DISPATCH(Float32)
|
||||
DISPATCH(Float64)
|
||||
DISPATCH(Decimal32)
|
||||
DISPATCH(Decimal64)
|
||||
DISPATCH(Decimal128)
|
||||
#undef DISPATCH
|
||||
else throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void TrieDictionary::getItemsImpl(
|
||||
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const
|
||||
|
||||
@ -218,10 +218,6 @@ private:
|
||||
Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
|
||||
|
||||
|
||||
template <typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void
|
||||
getItemsNumber(const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultGetter>
|
||||
void
|
||||
getItemsImpl(const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, DefaultGetter && get_default) const;
|
||||
|
||||
@ -130,7 +130,7 @@ void registerOutputFormatXML(FormatFactory & factory);
|
||||
void registerOutputFormatODBCDriver(FormatFactory & factory);
|
||||
void registerOutputFormatODBCDriver2(FormatFactory & factory);
|
||||
void registerOutputFormatNull(FormatFactory & factory);
|
||||
void registerOutputFormatMySQL(FormatFactory & factory);
|
||||
void registerOutputFormatMySQLWire(FormatFactory & factory);
|
||||
|
||||
/// Input only formats.
|
||||
|
||||
@ -169,7 +169,7 @@ FormatFactory::FormatFactory()
|
||||
registerOutputFormatODBCDriver(*this);
|
||||
registerOutputFormatODBCDriver2(*this);
|
||||
registerOutputFormatNull(*this);
|
||||
registerOutputFormatMySQL(*this);
|
||||
registerOutputFormatMySQLWire(*this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
#include <DataStreams/MySQLBlockOutputStream.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerOutputFormatMySQL(FormatFactory & factory)
|
||||
{
|
||||
factory.registerOutputFormat("MySQL", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
const FormatSettings &)
|
||||
{
|
||||
return std::make_shared<MySQLBlockOutputStream>(buf, sample, const_cast<Context &>(context));
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
#include "MySQLBlockOutputStream.h"
|
||||
#include "MySQLWireBlockOutputStream.h"
|
||||
#include <Core/MySQLProtocol.h>
|
||||
#include <Interpreters/ProcessList.h>
|
||||
#include <iomanip>
|
||||
@ -9,15 +9,15 @@ namespace DB
|
||||
|
||||
using namespace MySQLProtocol;
|
||||
|
||||
MySQLBlockOutputStream::MySQLBlockOutputStream(WriteBuffer & buf, const Block & header, Context & context)
|
||||
MySQLWireBlockOutputStream::MySQLWireBlockOutputStream(WriteBuffer & buf, const Block & header, Context & context)
|
||||
: header(header)
|
||||
, context(context)
|
||||
, packet_sender(new PacketSender(buf, context.sequence_id, "MySQLBlockOutputStream"))
|
||||
, packet_sender(std::make_shared<PacketSender>(buf, context.sequence_id))
|
||||
{
|
||||
packet_sender->max_packet_size = context.max_packet_size;
|
||||
}
|
||||
|
||||
void MySQLBlockOutputStream::writePrefix()
|
||||
void MySQLWireBlockOutputStream::writePrefix()
|
||||
{
|
||||
if (header.columns() == 0)
|
||||
return;
|
||||
@ -26,8 +26,7 @@ void MySQLBlockOutputStream::writePrefix()
|
||||
|
||||
for (const ColumnWithTypeAndName & column : header.getColumnsWithTypeAndName())
|
||||
{
|
||||
ColumnDefinition column_definition(column.name, CharacterSet::binary, std::numeric_limits<uint32_t>::max(),
|
||||
ColumnType::MYSQL_TYPE_STRING, 0, 0);
|
||||
ColumnDefinition column_definition(column.name, CharacterSet::binary, 0, ColumnType::MYSQL_TYPE_STRING, 0, 0);
|
||||
packet_sender->sendPacket(column_definition);
|
||||
}
|
||||
|
||||
@ -37,7 +36,7 @@ void MySQLBlockOutputStream::writePrefix()
|
||||
}
|
||||
}
|
||||
|
||||
void MySQLBlockOutputStream::write(const Block & block)
|
||||
void MySQLWireBlockOutputStream::write(const Block & block)
|
||||
{
|
||||
size_t rows = block.rows();
|
||||
|
||||
@ -57,7 +56,7 @@ void MySQLBlockOutputStream::write(const Block & block)
|
||||
}
|
||||
}
|
||||
|
||||
void MySQLBlockOutputStream::writeSuffix()
|
||||
void MySQLWireBlockOutputStream::writeSuffix()
|
||||
{
|
||||
QueryStatus * process_list_elem = context.getProcessListElement();
|
||||
CurrentThread::finalizePerformanceCounters();
|
||||
@ -79,7 +78,7 @@ void MySQLBlockOutputStream::writeSuffix()
|
||||
packet_sender->sendPacket(EOF_Packet(0, 0), true);
|
||||
}
|
||||
|
||||
void MySQLBlockOutputStream::flush()
|
||||
void MySQLWireBlockOutputStream::flush()
|
||||
{
|
||||
packet_sender->out->next();
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "IBlockOutputStream.h"
|
||||
#include <Core/MySQLProtocol.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -11,10 +11,10 @@ namespace DB
|
||||
|
||||
/** Interface for writing rows in MySQL Client/Server Protocol format.
|
||||
*/
|
||||
class MySQLBlockOutputStream : public IBlockOutputStream
|
||||
class MySQLWireBlockOutputStream : public IBlockOutputStream
|
||||
{
|
||||
public:
|
||||
MySQLBlockOutputStream(WriteBuffer & buf, const Block & header, Context & context);
|
||||
MySQLWireBlockOutputStream(WriteBuffer & buf, const Block & header, Context & context);
|
||||
|
||||
Block getHeader() const { return header; }
|
||||
|
||||
@ -31,6 +31,6 @@ private:
|
||||
FormatSettings format_settings;
|
||||
};
|
||||
|
||||
using MySQLBlockOutputStreamPtr = std::shared_ptr<MySQLBlockOutputStream>;
|
||||
using MySQLWireBlockOutputStreamPtr = std::shared_ptr<MySQLWireBlockOutputStream>;
|
||||
|
||||
}
|
||||
19
dbms/src/Formats/MySQLWireFormat.cpp
Normal file
19
dbms/src/Formats/MySQLWireFormat.cpp
Normal file
@ -0,0 +1,19 @@
|
||||
#include <Formats/MySQLWireBlockOutputStream.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerOutputFormatMySQLWire(FormatFactory & factory)
|
||||
{
|
||||
factory.registerOutputFormat("MySQLWire", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
const FormatSettings &)
|
||||
{
|
||||
return std::make_shared<MySQLWireBlockOutputStream>(buf, sample, const_cast<Context &>(context));
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -354,27 +354,27 @@ template <bool V, typename T> struct Case : std::bool_constant<V> { using type =
|
||||
template <typename... Ts> using Switch = typename std::disjunction<Ts..., Case<true, InvalidType>>::type;
|
||||
|
||||
template <typename DataType> constexpr bool IsIntegral = false;
|
||||
template <> constexpr bool IsIntegral<DataTypeUInt8> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeUInt16> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeUInt32> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeUInt64> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeInt8> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeInt16> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeInt32> = true;
|
||||
template <> constexpr bool IsIntegral<DataTypeInt64> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeUInt8> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeUInt16> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeUInt32> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeUInt64> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeInt8> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeInt16> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeInt32> = true;
|
||||
template <> inline constexpr bool IsIntegral<DataTypeInt64> = true;
|
||||
|
||||
template <typename DataType> constexpr bool IsFloatingPoint = false;
|
||||
template <> constexpr bool IsFloatingPoint<DataTypeFloat32> = true;
|
||||
template <> constexpr bool IsFloatingPoint<DataTypeFloat64> = true;
|
||||
template <> inline constexpr bool IsFloatingPoint<DataTypeFloat32> = true;
|
||||
template <> inline constexpr bool IsFloatingPoint<DataTypeFloat64> = true;
|
||||
|
||||
template <typename DataType> constexpr bool IsDateOrDateTime = false;
|
||||
template <> constexpr bool IsDateOrDateTime<DataTypeDate> = true;
|
||||
template <> constexpr bool IsDateOrDateTime<DataTypeDateTime> = true;
|
||||
template <> inline constexpr bool IsDateOrDateTime<DataTypeDate> = true;
|
||||
template <> inline constexpr bool IsDateOrDateTime<DataTypeDateTime> = true;
|
||||
|
||||
template <typename T0, typename T1> constexpr bool UseLeftDecimal = false;
|
||||
template <> constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal32>> = true;
|
||||
template <> constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal64>> = true;
|
||||
template <> constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal32>> = true;
|
||||
template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal32>> = true;
|
||||
template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal64>> = true;
|
||||
template <> inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal32>> = true;
|
||||
|
||||
template <typename T> using DataTypeFromFieldType = std::conditional_t<std::is_same_v<T, NumberTraits::Error>, InvalidType, DataTypeNumber<T>>;
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
@ -661,21 +662,77 @@ DECLARE_DICT_GET_TRAITS(UInt32, DataTypeDateTime)
|
||||
DECLARE_DICT_GET_TRAITS(UInt128, DataTypeUUID)
|
||||
#undef DECLARE_DICT_GET_TRAITS
|
||||
|
||||
template <typename T> struct DictGetTraits<DataTypeDecimal<T>>
|
||||
{
|
||||
static constexpr bool is_dec32 = std::is_same_v<T, Decimal32>;
|
||||
static constexpr bool is_dec64 = std::is_same_v<T, Decimal64>;
|
||||
static constexpr bool is_dec128 = std::is_same_v<T, Decimal128>;
|
||||
|
||||
template <typename DictionaryType>
|
||||
static void get(const DictionaryType * dict, const std::string & name, const PaddedPODArray<UInt64> & ids,
|
||||
DecimalPaddedPODArray<T> & out)
|
||||
{
|
||||
if constexpr (is_dec32) dict->getDecimal32(name, ids, out);
|
||||
if constexpr (is_dec64) dict->getDecimal64(name, ids, out);
|
||||
if constexpr (is_dec128) dict->getDecimal128(name, ids, out);
|
||||
}
|
||||
|
||||
template <typename DictionaryType>
|
||||
static void get(const DictionaryType * dict, const std::string & name, const Columns & key_columns, const DataTypes & key_types,
|
||||
DecimalPaddedPODArray<T> & out)
|
||||
{
|
||||
if constexpr (is_dec32) dict->getDecimal32(name, key_columns, key_types, out);
|
||||
if constexpr (is_dec64) dict->getDecimal64(name, key_columns, key_types, out);
|
||||
if constexpr (is_dec128) dict->getDecimal128(name, key_columns, key_types, out);
|
||||
}
|
||||
|
||||
template <typename DictionaryType>
|
||||
static void get(const DictionaryType * dict, const std::string & name, const PaddedPODArray<UInt64> & ids,
|
||||
const PaddedPODArray<Int64> & dates, DecimalPaddedPODArray<T> & out)
|
||||
{
|
||||
if constexpr (is_dec32) dict->getDecimal32(name, ids, dates, out);
|
||||
if constexpr (is_dec64) dict->getDecimal64(name, ids, dates, out);
|
||||
if constexpr (is_dec128) dict->getDecimal128(name, ids, dates, out);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename DefaultsType>
|
||||
static void getOrDefault(const DictionaryType * dict, const std::string & name, const PaddedPODArray<UInt64> & ids,
|
||||
const DefaultsType & def, DecimalPaddedPODArray<T> & out)
|
||||
{
|
||||
if constexpr (is_dec32) dict->getDecimal32(name, ids, def, out);
|
||||
if constexpr (is_dec64) dict->getDecimal64(name, ids, def, out);
|
||||
if constexpr (is_dec128) dict->getDecimal128(name, ids, def, out);
|
||||
}
|
||||
|
||||
template <typename DictionaryType, typename DefaultsType>
|
||||
static void getOrDefault(const DictionaryType * dict, const std::string & name, const Columns & key_columns,
|
||||
const DataTypes & key_types, const DefaultsType & def, DecimalPaddedPODArray<T> & out)
|
||||
{
|
||||
if constexpr (is_dec32) dict->getDecimal32(name, key_columns, key_types, def, out);
|
||||
if constexpr (is_dec64) dict->getDecimal64(name, key_columns, key_types, def, out);
|
||||
if constexpr (is_dec128) dict->getDecimal128(name, key_columns, key_types, def, out);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename DataType, typename Name>
|
||||
class FunctionDictGet final : public IFunction
|
||||
{
|
||||
using Type = typename DataType::FieldType;
|
||||
using ColVec = std::conditional_t<IsDecimalNumber<Type>, ColumnDecimal<Type>, ColumnVector<Type>>;
|
||||
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context & context)
|
||||
static FunctionPtr create(const Context & context, UInt32 dec_scale = 0)
|
||||
{
|
||||
return std::make_shared<FunctionDictGet>(context.getExternalDictionaries());
|
||||
return std::make_shared<FunctionDictGet>(context.getExternalDictionaries(), dec_scale);
|
||||
}
|
||||
|
||||
FunctionDictGet(const ExternalDictionaries & dictionaries) : dictionaries(dictionaries) {}
|
||||
FunctionDictGet(const ExternalDictionaries & dictionaries, UInt32 dec_scale = 0)
|
||||
: dictionaries(dictionaries)
|
||||
, decimal_scale(dec_scale)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -719,7 +776,10 @@ private:
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
}
|
||||
|
||||
return std::make_shared<DataType>();
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
return std::make_shared<DataType>(DataType::maxPrecision(), decimal_scale);
|
||||
else
|
||||
return std::make_shared<DataType>();
|
||||
}
|
||||
|
||||
bool isDeterministic() const override { return false; }
|
||||
@ -771,7 +831,11 @@ private:
|
||||
const auto id_col_untyped = block.getByPosition(arguments[2]).column.get();
|
||||
if (const auto id_col = checkAndGetColumn<ColumnUInt64>(id_col_untyped))
|
||||
{
|
||||
auto out = ColumnVector<Type>::create(id_col->size());
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(id_col->size(), decimal_scale);
|
||||
else
|
||||
out = ColVec::create(id_col->size());
|
||||
const auto & ids = id_col->getData();
|
||||
auto & data = out->getData();
|
||||
DictGetTraits<DataType>::get(dict, attr_name, ids, data);
|
||||
@ -780,9 +844,21 @@ private:
|
||||
else if (const auto id_col_const = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
|
||||
{
|
||||
const PaddedPODArray<UInt64> ids(1, id_col_const->getValue<UInt64>());
|
||||
PaddedPODArray<Type> data(1);
|
||||
DictGetTraits<DataType>::get(dict, attr_name, ids, data);
|
||||
block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col_const->size(), toField(data.front()));
|
||||
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
{
|
||||
DecimalPaddedPODArray<Type> data(1, decimal_scale);
|
||||
DictGetTraits<DataType>::get(dict, attr_name, ids, data);
|
||||
block.getByPosition(result).column =
|
||||
DataType(DataType::maxPrecision(), decimal_scale).createColumnConst(
|
||||
id_col_const->size(), toField(data.front(), decimal_scale));
|
||||
}
|
||||
else
|
||||
{
|
||||
PaddedPODArray<Type> data(1);
|
||||
DictGetTraits<DataType>::get(dict, attr_name, ids, data);
|
||||
block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col_const->size(), toField(data.front()));
|
||||
}
|
||||
}
|
||||
else
|
||||
throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
|
||||
@ -818,7 +894,11 @@ private:
|
||||
const auto & key_columns = static_cast<const ColumnTuple &>(*key_col).getColumnsCopy();
|
||||
const auto & key_types = static_cast<const DataTypeTuple &>(*key_col_with_type.type).getElements();
|
||||
|
||||
auto out = ColumnVector<Type>::create(key_columns.front()->size());
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(key_columns.front()->size(), decimal_scale);
|
||||
else
|
||||
out = ColVec::create(key_columns.front()->size());
|
||||
auto & data = out->getData();
|
||||
DictGetTraits<DataType>::get(dict, attr_name, key_columns, key_types, data);
|
||||
block.getByPosition(result).column = std::move(out);
|
||||
@ -855,7 +935,11 @@ private:
|
||||
const auto & id_col_values = getColumnDataAsPaddedPODArray(*id_col_untyped, id_col_values_storage);
|
||||
const auto & range_col_values = getColumnDataAsPaddedPODArray(*range_col_untyped, range_col_values_storage);
|
||||
|
||||
auto out = ColumnVector<Type>::create(id_col_untyped->size());
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(id_col_untyped->size(), decimal_scale);
|
||||
else
|
||||
out = ColVec::create(id_col_untyped->size());
|
||||
auto & data = out->getData();
|
||||
DictGetTraits<DataType>::get(dict, attr_name, id_col_values, range_col_values, data);
|
||||
block.getByPosition(result).column = std::move(out);
|
||||
@ -864,6 +948,7 @@ private:
|
||||
}
|
||||
|
||||
const ExternalDictionaries & dictionaries;
|
||||
UInt32 decimal_scale;
|
||||
};
|
||||
|
||||
struct NameDictGetUInt8 { static constexpr auto name = "dictGetUInt8"; };
|
||||
@ -879,6 +964,9 @@ struct NameDictGetFloat64 { static constexpr auto name = "dictGetFloat64"; };
|
||||
struct NameDictGetDate { static constexpr auto name = "dictGetDate"; };
|
||||
struct NameDictGetDateTime { static constexpr auto name = "dictGetDateTime"; };
|
||||
struct NameDictGetUUID { static constexpr auto name = "dictGetUUID"; };
|
||||
struct NameDictGetDecimal32 { static constexpr auto name = "dictGetDecimal32"; };
|
||||
struct NameDictGetDecimal64 { static constexpr auto name = "dictGetDecimal64"; };
|
||||
struct NameDictGetDecimal128 { static constexpr auto name = "dictGetDecimal128"; };
|
||||
|
||||
using FunctionDictGetUInt8 = FunctionDictGet<DataTypeUInt8, NameDictGetUInt8>;
|
||||
using FunctionDictGetUInt16 = FunctionDictGet<DataTypeUInt16, NameDictGetUInt16>;
|
||||
@ -893,22 +981,29 @@ using FunctionDictGetFloat64 = FunctionDictGet<DataTypeFloat64, NameDictGetFloat
|
||||
using FunctionDictGetDate = FunctionDictGet<DataTypeDate, NameDictGetDate>;
|
||||
using FunctionDictGetDateTime = FunctionDictGet<DataTypeDateTime, NameDictGetDateTime>;
|
||||
using FunctionDictGetUUID = FunctionDictGet<DataTypeUUID, NameDictGetUUID>;
|
||||
using FunctionDictGetDecimal32 = FunctionDictGet<DataTypeDecimal<Decimal32>, NameDictGetDecimal32>;
|
||||
using FunctionDictGetDecimal64 = FunctionDictGet<DataTypeDecimal<Decimal64>, NameDictGetDecimal64>;
|
||||
using FunctionDictGetDecimal128 = FunctionDictGet<DataTypeDecimal<Decimal128>, NameDictGetDecimal128>;
|
||||
|
||||
|
||||
template <typename DataType, typename Name>
|
||||
class FunctionDictGetOrDefault final : public IFunction
|
||||
{
|
||||
using Type = typename DataType::FieldType;
|
||||
using ColVec = std::conditional_t<IsDecimalNumber<Type>, ColumnDecimal<Type>, ColumnVector<Type>>;
|
||||
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context & context)
|
||||
static FunctionPtr create(const Context & context, UInt32 dec_scale = 0)
|
||||
{
|
||||
return std::make_shared<FunctionDictGetOrDefault>(context.getExternalDictionaries());
|
||||
return std::make_shared<FunctionDictGetOrDefault>(context.getExternalDictionaries(), dec_scale);
|
||||
}
|
||||
|
||||
FunctionDictGetOrDefault(const ExternalDictionaries & dictionaries) : dictionaries(dictionaries) {}
|
||||
FunctionDictGetOrDefault(const ExternalDictionaries & dictionaries, UInt32 dec_scale = 0)
|
||||
: dictionaries(dictionaries)
|
||||
, decimal_scale(dec_scale)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -935,9 +1030,12 @@ private:
|
||||
|
||||
if (!checkAndGetDataType<DataType>(arguments[3].get()))
|
||||
throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
|
||||
+ ", must be " + String(DataType{}.getFamilyName()) + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
+ ", must be " + TypeName<Type>::get() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
return std::make_shared<DataType>();
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
return std::make_shared<DataType>(DataType::maxPrecision(), decimal_scale);
|
||||
else
|
||||
return std::make_shared<DataType>();
|
||||
}
|
||||
|
||||
bool isDeterministic() const override { return false; }
|
||||
@ -999,20 +1097,28 @@ private:
|
||||
{
|
||||
const auto default_col_untyped = block.getByPosition(arguments[3]).column.get();
|
||||
|
||||
if (const auto default_col = checkAndGetColumn<ColumnVector<Type>>(default_col_untyped))
|
||||
if (const auto default_col = checkAndGetColumn<ColVec>(default_col_untyped))
|
||||
{
|
||||
/// vector ids, vector defaults
|
||||
auto out = ColumnVector<Type>::create(id_col->size());
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(id_col->size(), decimal_scale);
|
||||
else
|
||||
out = ColVec::create(id_col->size());
|
||||
const auto & ids = id_col->getData();
|
||||
auto & data = out->getData();
|
||||
const auto & defs = default_col->getData();
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, defs, data);
|
||||
block.getByPosition(result).column = std::move(out);
|
||||
}
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColumnVector<Type>>(default_col_untyped))
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColVec>(default_col_untyped))
|
||||
{
|
||||
/// vector ids, const defaults
|
||||
auto out = ColumnVector<Type>::create(id_col->size());
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(id_col->size(), decimal_scale);
|
||||
else
|
||||
out = ColVec::create(id_col->size());
|
||||
const auto & ids = id_col->getData();
|
||||
auto & data = out->getData();
|
||||
const auto def = default_col_const->template getValue<Type>();
|
||||
@ -1020,7 +1126,7 @@ private:
|
||||
block.getByPosition(result).column = std::move(out);
|
||||
}
|
||||
else
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + TypeName<Type>::get(), ErrorCodes::ILLEGAL_COLUMN};
|
||||
}
|
||||
|
||||
template <typename DictionaryType>
|
||||
@ -1030,7 +1136,7 @@ private:
|
||||
{
|
||||
const auto default_col_untyped = block.getByPosition(arguments[3]).column.get();
|
||||
|
||||
if (const auto default_col = checkAndGetColumn<ColumnVector<Type>>(default_col_untyped))
|
||||
if (const auto default_col = checkAndGetColumn<ColVec>(default_col_untyped))
|
||||
{
|
||||
/// const ids, vector defaults
|
||||
const PaddedPODArray<UInt64> ids(1, id_col->getValue<UInt64>());
|
||||
@ -1038,24 +1144,48 @@ private:
|
||||
dictionary->has(ids, flags);
|
||||
if (flags.front())
|
||||
{
|
||||
PaddedPODArray<Type> data(1);
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, Type(), data);
|
||||
block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col->size(), toField(data.front()));
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
{
|
||||
DecimalPaddedPODArray<Type> data(1, decimal_scale);
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, Type(), data);
|
||||
block.getByPosition(result).column =
|
||||
DataType(DataType::maxPrecision(), decimal_scale).createColumnConst(
|
||||
id_col->size(), toField(data.front(), decimal_scale));
|
||||
}
|
||||
else
|
||||
{
|
||||
PaddedPODArray<Type> data(1);
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, Type(), data);
|
||||
block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front()));
|
||||
}
|
||||
}
|
||||
else
|
||||
block.getByPosition(result).column = block.getByPosition(arguments[3]).column; // reuse the default column
|
||||
}
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColumnVector<Type>>(default_col_untyped))
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColVec>(default_col_untyped))
|
||||
{
|
||||
/// const ids, const defaults
|
||||
const PaddedPODArray<UInt64> ids(1, id_col->getValue<UInt64>());
|
||||
PaddedPODArray<Type> data(1);
|
||||
const auto & def = default_col_const->template getValue<Type>();
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, def, data);
|
||||
block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col->size(), toField(data.front()));
|
||||
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
{
|
||||
DecimalPaddedPODArray<Type> data(1, decimal_scale);
|
||||
const auto & def = default_col_const->template getValue<Type>();
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, def, data);
|
||||
block.getByPosition(result).column =
|
||||
DataType(DataType::maxPrecision(), decimal_scale).createColumnConst(
|
||||
id_col->size(), toField(data.front(), decimal_scale));
|
||||
}
|
||||
else
|
||||
{
|
||||
PaddedPODArray<Type> data(1);
|
||||
const auto & def = default_col_const->template getValue<Type>();
|
||||
DictGetTraits<DataType>::getOrDefault(dictionary, attr_name, ids, def, data);
|
||||
block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front()));
|
||||
}
|
||||
}
|
||||
else
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + TypeName<Type>::get(), ErrorCodes::ILLEGAL_COLUMN};
|
||||
}
|
||||
|
||||
template <typename DictionaryType>
|
||||
@ -1082,31 +1212,36 @@ private:
|
||||
|
||||
/// @todo detect when all key columns are constant
|
||||
const auto rows = key_col->size();
|
||||
auto out = ColumnVector<Type>::create(rows);
|
||||
typename ColVec::MutablePtr out;
|
||||
if constexpr (IsDataTypeDecimal<DataType>)
|
||||
out = ColVec::create(rows, decimal_scale);
|
||||
else
|
||||
out = ColVec::create(rows);
|
||||
auto & data = out->getData();
|
||||
|
||||
const auto default_col_untyped = block.getByPosition(arguments[3]).column.get();
|
||||
if (const auto default_col = checkAndGetColumn<ColumnVector<Type>>(default_col_untyped))
|
||||
if (const auto default_col = checkAndGetColumn<ColVec>(default_col_untyped))
|
||||
{
|
||||
/// const defaults
|
||||
const auto & defs = default_col->getData();
|
||||
|
||||
DictGetTraits<DataType>::getOrDefault(dict, attr_name, key_columns, key_types, defs, data);
|
||||
}
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColumnVector<Type>>(default_col_untyped))
|
||||
else if (const auto default_col_const = checkAndGetColumnConst<ColVec>(default_col_untyped))
|
||||
{
|
||||
const auto def = default_col_const->template getValue<Type>();
|
||||
|
||||
DictGetTraits<DataType>::getOrDefault(dict, attr_name, key_columns, key_types, def, data);
|
||||
}
|
||||
else
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
|
||||
throw Exception{"Fourth argument of function " + getName() + " must be " + TypeName<Type>::get(), ErrorCodes::ILLEGAL_COLUMN};
|
||||
|
||||
block.getByPosition(result).column = std::move(out);
|
||||
return true;
|
||||
}
|
||||
|
||||
const ExternalDictionaries & dictionaries;
|
||||
UInt32 decimal_scale;
|
||||
};
|
||||
|
||||
struct NameDictGetUInt8OrDefault { static constexpr auto name = "dictGetUInt8OrDefault"; };
|
||||
@ -1122,6 +1257,9 @@ struct NameDictGetFloat64OrDefault { static constexpr auto name = "dictGetFloat6
|
||||
struct NameDictGetDateOrDefault { static constexpr auto name = "dictGetDateOrDefault"; };
|
||||
struct NameDictGetDateTimeOrDefault { static constexpr auto name = "dictGetDateTimeOrDefault"; };
|
||||
struct NameDictGetUUIDOrDefault { static constexpr auto name = "dictGetUUIDOrDefault"; };
|
||||
struct NameDictGetDecimal32OrDefault { static constexpr auto name = "dictGetDecimal32OrDefault"; };
|
||||
struct NameDictGetDecimal64OrDefault { static constexpr auto name = "dictGetDecimal64OrDefault"; };
|
||||
struct NameDictGetDecimal128OrDefault { static constexpr auto name = "dictGetDecimal128OrDefault"; };
|
||||
|
||||
using FunctionDictGetUInt8OrDefault = FunctionDictGetOrDefault<DataTypeUInt8, NameDictGetUInt8OrDefault>;
|
||||
using FunctionDictGetUInt16OrDefault = FunctionDictGetOrDefault<DataTypeUInt16, NameDictGetUInt16OrDefault>;
|
||||
@ -1136,21 +1274,10 @@ using FunctionDictGetFloat64OrDefault = FunctionDictGetOrDefault<DataTypeFloat64
|
||||
using FunctionDictGetDateOrDefault = FunctionDictGetOrDefault<DataTypeDate, NameDictGetDateOrDefault>;
|
||||
using FunctionDictGetDateTimeOrDefault = FunctionDictGetOrDefault<DataTypeDateTime, NameDictGetDateTimeOrDefault>;
|
||||
using FunctionDictGetUUIDOrDefault = FunctionDictGetOrDefault<DataTypeUUID, NameDictGetUUIDOrDefault>;
|
||||
using FunctionDictGetDecimal32OrDefault = FunctionDictGetOrDefault<DataTypeDecimal<Decimal32>, NameDictGetDecimal32OrDefault>;
|
||||
using FunctionDictGetDecimal64OrDefault = FunctionDictGetOrDefault<DataTypeDecimal<Decimal64>, NameDictGetDecimal64OrDefault>;
|
||||
using FunctionDictGetDecimal128OrDefault = FunctionDictGetOrDefault<DataTypeDecimal<Decimal128>, NameDictGetDecimal128OrDefault>;
|
||||
|
||||
#define FOR_DICT_TYPES(M) \
|
||||
M(UInt8) \
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(Int8) \
|
||||
M(Int16) \
|
||||
M(Int32) \
|
||||
M(Int64) \
|
||||
M(Float32) \
|
||||
M(Float64) \
|
||||
M(Date) \
|
||||
M(DateTime) \
|
||||
M(UUID)
|
||||
|
||||
/// This variant of function derives the result type automatically.
|
||||
class FunctionDictGetNoType final : public IFunction
|
||||
@ -1225,15 +1352,63 @@ private:
|
||||
if (attribute.name == attr_name)
|
||||
{
|
||||
WhichDataType dt = attribute.type;
|
||||
if (dt.idx == TypeIndex::String)
|
||||
impl = FunctionDictGetString::create(context);
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (dt.idx == TypeIndex::TYPE) \
|
||||
impl = FunctionDictGet<DataType##TYPE, NameDictGet##TYPE>::create(context);
|
||||
FOR_DICT_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
else
|
||||
throw Exception("Unknown dictGet type", ErrorCodes::UNKNOWN_TYPE);
|
||||
switch (dt.idx)
|
||||
{
|
||||
case TypeIndex::String:
|
||||
case TypeIndex::FixedString:
|
||||
impl = FunctionDictGetString::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt8:
|
||||
impl = FunctionDictGetUInt8::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt16:
|
||||
impl = FunctionDictGetUInt16::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt32:
|
||||
impl = FunctionDictGetUInt32::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt64:
|
||||
impl = FunctionDictGetUInt64::create(context);
|
||||
break;
|
||||
case TypeIndex::Int8:
|
||||
impl = FunctionDictGetInt8::create(context);
|
||||
break;
|
||||
case TypeIndex::Int16:
|
||||
impl = FunctionDictGetInt16::create(context);
|
||||
break;
|
||||
case TypeIndex::Int32:
|
||||
impl = FunctionDictGetInt32::create(context);
|
||||
break;
|
||||
case TypeIndex::Int64:
|
||||
impl = FunctionDictGetInt64::create(context);
|
||||
break;
|
||||
case TypeIndex::Float32:
|
||||
impl = FunctionDictGetFloat32::create(context);
|
||||
break;
|
||||
case TypeIndex::Float64:
|
||||
impl = FunctionDictGetFloat64::create(context);
|
||||
break;
|
||||
case TypeIndex::Date:
|
||||
impl = FunctionDictGetDate::create(context);
|
||||
break;
|
||||
case TypeIndex::DateTime:
|
||||
impl = FunctionDictGetDateTime::create(context);
|
||||
break;
|
||||
case TypeIndex::UUID:
|
||||
impl = FunctionDictGetUUID::create(context);
|
||||
break;
|
||||
case TypeIndex::Decimal32:
|
||||
impl = FunctionDictGetDecimal32::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
case TypeIndex::Decimal64:
|
||||
impl = FunctionDictGetDecimal64::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
case TypeIndex::Decimal128:
|
||||
impl = FunctionDictGetDecimal128::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
default:
|
||||
throw Exception("Unknown dictGet type", ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
return attribute.type;
|
||||
}
|
||||
}
|
||||
@ -1312,26 +1487,70 @@ private:
|
||||
const DictionaryAttribute & attribute = structure.attributes[idx];
|
||||
if (attribute.name == attr_name)
|
||||
{
|
||||
auto arg_type = arguments[3].type;
|
||||
WhichDataType dt = attribute.type;
|
||||
if (dt.idx == TypeIndex::String)
|
||||
|
||||
if ((arg_type->getTypeId() != dt.idx) || (dt.isStringOrFixedString() && !isString(arg_type)))
|
||||
throw Exception{"Illegal type " + arg_type->getName() + " of fourth argument of function " + getName() +
|
||||
", must be " + getTypeName(dt.idx) + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
|
||||
switch (dt.idx)
|
||||
{
|
||||
if (!isString(arguments[3].type))
|
||||
throw Exception{"Illegal type " + arguments[3].type->getName() + " of fourth argument of function " + getName() +
|
||||
", must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
impl = FunctionDictGetStringOrDefault::create(context);
|
||||
case TypeIndex::String:
|
||||
impl = FunctionDictGetStringOrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt8:
|
||||
impl = FunctionDictGetUInt8OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt16:
|
||||
impl = FunctionDictGetUInt16OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt32:
|
||||
impl = FunctionDictGetUInt32OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::UInt64:
|
||||
impl = FunctionDictGetUInt64OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Int8:
|
||||
impl = FunctionDictGetInt8OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Int16:
|
||||
impl = FunctionDictGetInt16OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Int32:
|
||||
impl = FunctionDictGetInt32OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Int64:
|
||||
impl = FunctionDictGetInt64OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Float32:
|
||||
impl = FunctionDictGetFloat32OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Float64:
|
||||
impl = FunctionDictGetFloat64OrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Date:
|
||||
impl = FunctionDictGetDateOrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::DateTime:
|
||||
impl = FunctionDictGetDateTimeOrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::UUID:
|
||||
impl = FunctionDictGetUUIDOrDefault::create(context);
|
||||
break;
|
||||
case TypeIndex::Decimal32:
|
||||
impl = FunctionDictGetDecimal32OrDefault::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
case TypeIndex::Decimal64:
|
||||
impl = FunctionDictGetDecimal64OrDefault::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
case TypeIndex::Decimal128:
|
||||
impl = FunctionDictGetDecimal128OrDefault::create(context, getDecimalScale(*attribute.type));
|
||||
break;
|
||||
default:
|
||||
throw Exception("Unknown dictGetOrDefault type", ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
#define DISPATCH(TYPE) \
|
||||
else if (dt.idx == TypeIndex::TYPE) \
|
||||
{ \
|
||||
if (!checkAndGetDataType<DataType##TYPE>(arguments[3].type.get())) \
|
||||
throw Exception{"Illegal type " + arguments[3].type->getName() + " of fourth argument of function " + getName() \
|
||||
+ ", must be " + String(DataType##TYPE{}.getFamilyName()) + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; \
|
||||
impl = FunctionDictGetOrDefault<DataType##TYPE, NameDictGet##TYPE ## OrDefault>::create(context); \
|
||||
}
|
||||
FOR_DICT_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
else
|
||||
throw Exception("Unknown dictGetOrDefault type", ErrorCodes::UNKNOWN_TYPE);
|
||||
|
||||
return attribute.type;
|
||||
}
|
||||
}
|
||||
|
||||
@ -120,11 +120,22 @@ private:
|
||||
/// prepare() does Impl-specific preparation before handling each row.
|
||||
impl.prepare(Name::name, block, arguments, result_pos);
|
||||
|
||||
bool json_parsed_ok = false;
|
||||
if (col_json_const)
|
||||
{
|
||||
StringRef json{reinterpret_cast<const char *>(&chars[0]), offsets[0] - 1};
|
||||
json_parsed_ok = parser.parse(json);
|
||||
}
|
||||
|
||||
for (const auto i : ext::range(0, input_rows_count))
|
||||
{
|
||||
StringRef json{reinterpret_cast<const char *>(&chars[offsets[i - 1]]), offsets[i] - offsets[i - 1] - 1};
|
||||
bool ok = parser.parse(json);
|
||||
if (!col_json_const)
|
||||
{
|
||||
StringRef json{reinterpret_cast<const char *>(&chars[offsets[i - 1]]), offsets[i] - offsets[i - 1] - 1};
|
||||
json_parsed_ok = parser.parse(json);
|
||||
}
|
||||
|
||||
bool ok = json_parsed_ok;
|
||||
if (ok)
|
||||
{
|
||||
auto it = parser.getRoot();
|
||||
|
||||
@ -479,22 +479,22 @@ struct NameNgramDistanceUTF8CaseInsensitive
|
||||
static constexpr auto name = "ngramDistanceCaseInsensitiveUTF8";
|
||||
};
|
||||
|
||||
struct NameNgramEntry
|
||||
struct NameNgramSearch
|
||||
{
|
||||
static constexpr auto name = "ngramEntry";
|
||||
static constexpr auto name = "ngramSearch";
|
||||
};
|
||||
struct NameNgramEntryCaseInsensitive
|
||||
struct NameNgramSearchCaseInsensitive
|
||||
{
|
||||
static constexpr auto name = "ngramEntryCaseInsensitive";
|
||||
static constexpr auto name = "ngramSearchCaseInsensitive";
|
||||
};
|
||||
struct NameNgramEntryUTF8
|
||||
struct NameNgramSearchUTF8
|
||||
{
|
||||
static constexpr auto name = "ngramEntryUTF8";
|
||||
static constexpr auto name = "ngramSearchUTF8";
|
||||
};
|
||||
|
||||
struct NameNgramEntryUTF8CaseInsensitive
|
||||
struct NameNgramSearchUTF8CaseInsensitive
|
||||
{
|
||||
static constexpr auto name = "ngramEntryCaseInsensitiveUTF8";
|
||||
static constexpr auto name = "ngramSearchCaseInsensitiveUTF8";
|
||||
};
|
||||
|
||||
using FunctionNgramDistance = FunctionsStringSimilarity<NgramDistanceImpl<4, UInt8, false, false, true>, NameNgramDistance>;
|
||||
@ -502,10 +502,10 @@ using FunctionNgramDistanceCaseInsensitive = FunctionsStringSimilarity<NgramDist
|
||||
using FunctionNgramDistanceUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, false, true>, NameNgramDistanceUTF8>;
|
||||
using FunctionNgramDistanceCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, true>, NameNgramDistanceUTF8CaseInsensitive>;
|
||||
|
||||
using FunctionNgramEntry = FunctionsStringSimilarity<NgramDistanceImpl<4, UInt8, false, false, false>, NameNgramEntry>;
|
||||
using FunctionNgramEntryCaseInsensitive = FunctionsStringSimilarity<NgramDistanceImpl<4, UInt8, false, true, false>, NameNgramEntryCaseInsensitive>;
|
||||
using FunctionNgramEntryUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, false, false>, NameNgramEntryUTF8>;
|
||||
using FunctionNgramEntryCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, false>, NameNgramEntryUTF8CaseInsensitive>;
|
||||
using FunctionNgramSearch = FunctionsStringSimilarity<NgramDistanceImpl<4, UInt8, false, false, false>, NameNgramSearch>;
|
||||
using FunctionNgramSearchCaseInsensitive = FunctionsStringSimilarity<NgramDistanceImpl<4, UInt8, false, true, false>, NameNgramSearchCaseInsensitive>;
|
||||
using FunctionNgramSearchUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, false, false>, NameNgramSearchUTF8>;
|
||||
using FunctionNgramSearchCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, false>, NameNgramSearchUTF8CaseInsensitive>;
|
||||
|
||||
|
||||
void registerFunctionsStringSimilarity(FunctionFactory & factory)
|
||||
@ -515,10 +515,10 @@ void registerFunctionsStringSimilarity(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionNgramDistanceUTF8>();
|
||||
factory.registerFunction<FunctionNgramDistanceCaseInsensitiveUTF8>();
|
||||
|
||||
factory.registerFunction<FunctionNgramEntry>();
|
||||
factory.registerFunction<FunctionNgramEntryCaseInsensitive>();
|
||||
factory.registerFunction<FunctionNgramEntryUTF8>();
|
||||
factory.registerFunction<FunctionNgramEntryCaseInsensitiveUTF8>();
|
||||
factory.registerFunction<FunctionNgramSearch>();
|
||||
factory.registerFunction<FunctionNgramSearchCaseInsensitive>();
|
||||
factory.registerFunction<FunctionNgramSearchUTF8>();
|
||||
factory.registerFunction<FunctionNgramSearchCaseInsensitiveUTF8>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnAggregateFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
@ -60,7 +61,7 @@ public:
|
||||
throw Exception("Argument for function " + getName() + " must have type AggregateFunction - state of aggregate function.",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return type->getReturnType();
|
||||
return type->getReturnTypeToPredict();
|
||||
}
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
|
||||
|
||||
@ -21,10 +21,23 @@ set (INTERNAL_COMPILER_NO_WARNING OFF CACHE INTERNAL "")
|
||||
set (INTERNAL_COMPILER_HEADERS_DIR "headers" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_HEADERS_RELATIVE "${INTERNAL_COMPILER_HEADERS_DIR}/${VERSION_STRING}" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_HEADERS "${PATH_SHARE}/clickhouse/${INTERNAL_COMPILER_HEADERS_RELATIVE}" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING "")
|
||||
set (INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "")
|
||||
|
||||
set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -nostdinc -nostdinc++ -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}" CACHE STRING "")
|
||||
if(OS_FREEBSD)
|
||||
set(INTERNAL_COMPILER_HEADERS_ROOT "" CACHE STRING "")
|
||||
else()
|
||||
set(INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING "")
|
||||
set(INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "")
|
||||
endif()
|
||||
|
||||
if(NOT INTERNAL_COMPILER_FLAGS)
|
||||
set(INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse" CACHE STRING "")
|
||||
if(INTERNAL_COMPILER_CUSTOM_ROOT)
|
||||
set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -nostdinc -nostdinc++")
|
||||
if(INTERNAL_COMPILER_HEADERS_ROOT)
|
||||
set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
# TODO: use libs from package: -nodefaultlibs -lm -lc -lgcc_s -lgcc -lc++ -lc++abi
|
||||
|
||||
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
|
||||
@ -47,7 +60,7 @@ string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Boost_INCLUDE_DIRS ${Boost_
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Foundation_INCLUDE_DIR ${Poco_Foundation_INCLUDE_DIR})
|
||||
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Util_INCLUDE_DIR ${Poco_Util_INCLUDE_DIR})
|
||||
|
||||
message (STATUS "Using internal=${USE_INTERNAL_LLVM_LIBRARY} compiler=${USE_EMBEDDED_COMPILER}: headers=${INTERNAL_COMPILER_HEADERS} : ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}")
|
||||
message (STATUS "Using internal=${USE_INTERNAL_LLVM_LIBRARY} compiler=${USE_EMBEDDED_COMPILER}: headers=${INTERNAL_COMPILER_HEADERS} root=${INTERNAL_COMPILER_HEADERS_ROOT}: ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}")
|
||||
|
||||
set (CONFIG_COMPILE ${ClickHouse_BINARY_DIR}/dbms/src/Interpreters/config_compile.h)
|
||||
configure_file (${ClickHouse_SOURCE_DIR}/dbms/src/Interpreters/config_compile.h.in ${CONFIG_COMPILE})
|
||||
|
||||
@ -971,6 +971,10 @@ void DDLWorker::runMainThread()
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (e.code == Coordination::ZNONODE)
|
||||
{
|
||||
LOG_ERROR(log, "ZooKeeper error: " << getCurrentExceptionMessage(true));
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_ERROR(log, "Unexpected ZooKeeper error: " << getCurrentExceptionMessage(true) << ". Terminating.");
|
||||
|
||||
@ -38,6 +38,8 @@
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
|
||||
#include <Databases/DatabaseFactory.h>
|
||||
#include <Databases/IDatabase.h>
|
||||
@ -65,6 +67,7 @@ namespace ErrorCodes
|
||||
extern const int QUERY_IS_PROHIBITED;
|
||||
extern const int THERE_IS_NO_DEFAULT_VALUE;
|
||||
extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
|
||||
extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY;
|
||||
}
|
||||
|
||||
|
||||
@ -546,6 +549,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
|
||||
/// Set and retrieve list of columns.
|
||||
ColumnsDescription columns = setColumns(create, as_select_sample, as_storage);
|
||||
|
||||
/// Check low cardinality types in creating table if it was not allowed in setting
|
||||
if (!create.attach && !context.getSettingsRef().allow_suspicious_low_cardinality_types)
|
||||
{
|
||||
for (const auto & name_and_type_pair : columns.getAllPhysical())
|
||||
{
|
||||
if (const auto * current_type_ptr = typeid_cast<const DataTypeLowCardinality *>(name_and_type_pair.type.get()))
|
||||
{
|
||||
if (!isStringOrFixedString(*removeNullable(current_type_ptr->getDictionaryType())))
|
||||
throw Exception("Creating columns of type " + current_type_ptr->getName() + " is prohibited by default due to expected negative impact on performance. It can be enabled with the \"allow_suspicious_low_cardinality_types\" setting.",
|
||||
ErrorCodes::SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the table engine if it was not specified explicitly.
|
||||
setEngine(create);
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <DataStreams/NullBlockInputStream.h>
|
||||
#include <DataStreams/ConcatBlockInputStream.h>
|
||||
#include <DataStreams/ConvertingBlockInputStream.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
#include <Columns/getLeastSuperColumn.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
@ -114,39 +114,13 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(
|
||||
|
||||
for (size_t column_num = 0; column_num < num_columns; ++column_num)
|
||||
{
|
||||
std::vector<const ColumnWithTypeAndName *> columns;
|
||||
columns.reserve(num_selects);
|
||||
for (size_t i = 0; i < num_selects; ++i)
|
||||
columns.push_back(&headers[i].getByPosition(column_num));
|
||||
|
||||
ColumnWithTypeAndName & result_elem = result_header.getByPosition(column_num);
|
||||
|
||||
/// Determine common type.
|
||||
|
||||
DataTypes types(num_selects);
|
||||
for (size_t query_num = 0; query_num < num_selects; ++query_num)
|
||||
types[query_num] = headers[query_num].getByPosition(column_num).type;
|
||||
|
||||
result_elem.type = getLeastSupertype(types);
|
||||
|
||||
/// If there are different constness or different values of constants, the result must be non-constant.
|
||||
|
||||
if (result_elem.column->isColumnConst())
|
||||
{
|
||||
bool need_materialize = false;
|
||||
for (size_t query_num = 1; query_num < num_selects; ++query_num)
|
||||
{
|
||||
const ColumnWithTypeAndName & source_elem = headers[query_num].getByPosition(column_num);
|
||||
|
||||
if (!source_elem.column->isColumnConst()
|
||||
|| (static_cast<const ColumnConst &>(*result_elem.column).getField()
|
||||
!= static_cast<const ColumnConst &>(*source_elem.column).getField()))
|
||||
{
|
||||
need_materialize = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (need_materialize)
|
||||
result_elem.column = result_elem.type->createColumn();
|
||||
}
|
||||
|
||||
/// BTW, result column names are from first SELECT.
|
||||
result_elem = getLeastSuperColumn(columns);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,11 +7,18 @@
|
||||
#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@"
|
||||
#cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@"
|
||||
#cmakedefine INTERNAL_COMPILER_ENV "@INTERNAL_COMPILER_ENV@"
|
||||
#ifndef INTERNAL_COMPILER_ENV
|
||||
#define INTERNAL_COMPILER_ENV ""
|
||||
#if !defined(INTERNAL_COMPILER_ENV)
|
||||
# define INTERNAL_COMPILER_ENV ""
|
||||
#endif
|
||||
#cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@"
|
||||
#if !defined(INTERNAL_COMPILER_HEADERS)
|
||||
# define INTERNAL_COMPILER_HEADERS ""
|
||||
#endif
|
||||
#cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@"
|
||||
#if !defined(INTERNAL_COMPILER_HEADERS_ROOT)
|
||||
# define INTERNAL_COMPILER_HEADERS_ROOT ""
|
||||
#endif
|
||||
|
||||
#cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT
|
||||
#cmakedefine INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR "@INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR@"
|
||||
#cmakedefine INTERNAL_Poco_Foundation_INCLUDE_DIR "@INTERNAL_Poco_Foundation_INCLUDE_DIR@"
|
||||
|
||||
@ -31,7 +31,10 @@ ColumnDefaultKind columnDefaultKindFromString(const std::string & str)
|
||||
};
|
||||
|
||||
const auto it = map.find(str);
|
||||
return it != std::end(map) ? it->second : throw Exception{"Unknown column default specifier: " + str, ErrorCodes::LOGICAL_ERROR};
|
||||
if (it != std::end(map))
|
||||
return it->second;
|
||||
|
||||
throw Exception{"Unknown column default specifier: " + str, ErrorCodes::LOGICAL_ERROR};
|
||||
}
|
||||
|
||||
|
||||
@ -44,7 +47,10 @@ std::string toString(const ColumnDefaultKind kind)
|
||||
};
|
||||
|
||||
const auto it = map.find(kind);
|
||||
return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultKind", ErrorCodes::LOGICAL_ERROR};
|
||||
if (it != std::end(map))
|
||||
return it->second;
|
||||
|
||||
throw Exception{"Invalid ColumnDefaultKind", ErrorCodes::LOGICAL_ERROR};
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2831,6 +2831,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String &
|
||||
String backup_part_absolute_path = part_absolute_path;
|
||||
backup_part_absolute_path.replace(0, clickhouse_path.size(), backup_path);
|
||||
localBackup(part_absolute_path, backup_part_absolute_path);
|
||||
part->is_frozen = true;
|
||||
++parts_processed;
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
|
||||
#include "MergeTreeDataMergerMutator.h"
|
||||
|
||||
#include <Storages/MergeTree/MergeTreeSequentialBlockInputStream.h>
|
||||
#include <Storages/MergeTree/MergedBlockOutputStream.h>
|
||||
#include <Storages/MergeTree/DiskSpaceMonitor.h>
|
||||
@ -25,12 +26,9 @@
|
||||
#include <Common/SimpleIncrement.h>
|
||||
#include <Common/interpolate.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/localBackup.h>
|
||||
#include <Common/createHardLink.h>
|
||||
|
||||
#include <Poco/File.h>
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <iomanip>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <optional>
|
||||
#include "MergeTreeDataPart.h"
|
||||
|
||||
#include <optional>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Compression/CompressedReadBuffer.h>
|
||||
@ -14,13 +15,10 @@
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/localBackup.h>
|
||||
#include <Compression/CompressionInfo.h>
|
||||
#include <Storages/MergeTree/MergeTreeDataPart.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
|
||||
#include <Poco/File.h>
|
||||
#include <Poco/Path.h>
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/JSON.h>
|
||||
|
||||
@ -728,7 +726,8 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
|
||||
void MergeTreeDataPart::loadColumns(bool require)
|
||||
{
|
||||
String path = getFullPath() + "columns.txt";
|
||||
if (!Poco::File(path).exists())
|
||||
Poco::File poco_file_path{path};
|
||||
if (!poco_file_path.exists())
|
||||
{
|
||||
if (require)
|
||||
throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
|
||||
@ -750,6 +749,8 @@ void MergeTreeDataPart::loadColumns(bool require)
|
||||
return;
|
||||
}
|
||||
|
||||
is_frozen = !poco_file_path.canWrite();
|
||||
|
||||
ReadBufferFromFile file = openForReading(path);
|
||||
columns.readText(file);
|
||||
}
|
||||
|
||||
@ -106,6 +106,9 @@ struct MergeTreeDataPart
|
||||
/// If true it means that there are no ZooKeeper node for this part, so it should be deleted only from filesystem
|
||||
bool is_duplicate = false;
|
||||
|
||||
/// Frozen by ALTER TABLE ... FREEZE ...
|
||||
mutable bool is_frozen = false;
|
||||
|
||||
/**
|
||||
* Part state is a stage of its lifetime. States are ordered and state of a part could be increased only.
|
||||
* Part state should be modified under data_parts mutex.
|
||||
|
||||
@ -84,7 +84,7 @@ ASTPtr rewriteSelectQuery(const ASTPtr & query, const std::string & database, co
|
||||
/// The columns list in the original INSERT query is incorrect because inserted blocks are transformed
|
||||
/// to the form of the sample block of the Distributed table. So we rewrite it and add all columns from
|
||||
/// the sample block instead.
|
||||
ASTPtr createInsertToRemoteTableQuery(const std::string & database, const std::string & table, const Block & sample_block)
|
||||
ASTPtr createInsertToRemoteTableQuery(const std::string & database, const std::string & table, const Block & sample_block_non_materialized)
|
||||
{
|
||||
auto query = std::make_shared<ASTInsertQuery>();
|
||||
query->database = database;
|
||||
@ -93,7 +93,7 @@ ASTPtr createInsertToRemoteTableQuery(const std::string & database, const std::s
|
||||
auto columns = std::make_shared<ASTExpressionList>();
|
||||
query->columns = columns;
|
||||
query->children.push_back(columns);
|
||||
for (const auto & col : sample_block)
|
||||
for (const auto & col : sample_block_non_materialized)
|
||||
columns->children.push_back(std::make_shared<ASTIdentifier>(col.name));
|
||||
|
||||
return query;
|
||||
@ -333,7 +333,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c
|
||||
|
||||
/// DistributedBlockOutputStream will not own cluster, but will own ConnectionPools of the cluster
|
||||
return std::make_shared<DistributedBlockOutputStream>(
|
||||
context, *this, createInsertToRemoteTableQuery(remote_database, remote_table, getSampleBlock()), cluster,
|
||||
context, *this, createInsertToRemoteTableQuery(remote_database, remote_table, getSampleBlockNonMaterialized()), cluster,
|
||||
insert_sync, timeout);
|
||||
}
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ public:
|
||||
|
||||
private:
|
||||
Block sample_block;
|
||||
const Names & key_names;
|
||||
const Names key_names;
|
||||
bool use_nulls;
|
||||
SizeLimits limits;
|
||||
ASTTableJoin::Kind kind; /// LEFT | INNER ...
|
||||
|
||||
@ -1,30 +1,24 @@
|
||||
#include <Databases/IDatabase.h>
|
||||
#include "StorageMergeTree.h"
|
||||
|
||||
#include <Databases/IDatabase.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/localBackup.h>
|
||||
|
||||
#include <Interpreters/InterpreterAlterQuery.h>
|
||||
#include <Interpreters/PartLog.h>
|
||||
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <Storages/MergeTree/ActiveDataPartSet.h>
|
||||
#include <Storages/AlterCommands.h>
|
||||
#include <Storages/PartitionCommands.h>
|
||||
#include <Storages/StorageMergeTree.h>
|
||||
#include <Storages/MergeTree/MergeTreeBlockOutputStream.h>
|
||||
#include <Storages/MergeTree/DiskSpaceMonitor.h>
|
||||
#include <Storages/MergeTree/MergeList.h>
|
||||
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Poco/File.h>
|
||||
|
||||
#include <optional>
|
||||
|
||||
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
#include "StorageSystemParts.h"
|
||||
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
@ -5,7 +7,6 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <Storages/System/StorageSystemParts.h>
|
||||
#include <Storages/VirtualColumnUtils.h>
|
||||
#include <Databases/IDatabase.h>
|
||||
|
||||
@ -38,6 +39,7 @@ StorageSystemParts::StorageSystemParts(const std::string & name)
|
||||
{"data_version", std::make_shared<DataTypeUInt64>()},
|
||||
{"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
|
||||
{"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
|
||||
{"is_frozen", std::make_shared<DataTypeUInt8>()},
|
||||
|
||||
{"database", std::make_shared<DataTypeString>()},
|
||||
{"table", std::make_shared<DataTypeString>()},
|
||||
@ -96,6 +98,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
|
||||
columns[i++]->insert(static_cast<UInt64>(part->info.getDataVersion()));
|
||||
columns[i++]->insert(part->getIndexSizeInBytes());
|
||||
columns[i++]->insert(part->getIndexSizeInAllocatedBytes());
|
||||
columns[i++]->insert(part->is_frozen);
|
||||
|
||||
columns[i++]->insert(info.database);
|
||||
columns[i++]->insert(info.table);
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
#include "StorageSystemPartsColumns.h"
|
||||
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
@ -5,7 +7,6 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <Storages/System/StorageSystemPartsColumns.h>
|
||||
#include <Storages/VirtualColumnUtils.h>
|
||||
#include <Databases/IDatabase.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
#include <Storages/MergeTree/ReplicatedMergeTreePartHeader.h>
|
||||
#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
@ -57,6 +57,9 @@ if [ "$DATA_DIR_PATTERN" != "$DATA_DIR" ]; then
|
||||
export CLICKHOUSE_CONFIG=$DATA_DIR/etc/server-config.xml
|
||||
cp $CLICKHOUSE_CONFIG_USERS $DATA_DIR/etc
|
||||
cp -R -L $CLICKHOUSE_CONFIG_USERS_D $DATA_DIR/etc
|
||||
cat ${CONFIG_SERVER_DIR}/ints_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/ints_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/strings_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/strings_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/decimals_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/decimals_dictionary.xml
|
||||
fi
|
||||
|
||||
CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="${CLICKHOUSE_EXTRACT} --config=$CLICKHOUSE_CONFIG"}
|
||||
|
||||
1
dbms/tests/decimals_dictionary.xml
Symbolic link
1
dbms/tests/decimals_dictionary.xml
Symbolic link
@ -0,0 +1 @@
|
||||
../../docker/test/stateless/decimals_dictionary.xml
|
||||
@ -1,7 +1,7 @@
|
||||
version: '2.2'
|
||||
services:
|
||||
zoo1:
|
||||
image: zookeeper
|
||||
image: zookeeper:3.4.12
|
||||
restart: always
|
||||
environment:
|
||||
ZOO_TICK_TIME: 500
|
||||
@ -9,7 +9,7 @@ services:
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
|
||||
|
||||
zoo2:
|
||||
image: zookeeper
|
||||
image: zookeeper:3.4.12
|
||||
restart: always
|
||||
environment:
|
||||
ZOO_TICK_TIME: 500
|
||||
@ -17,7 +17,7 @@ services:
|
||||
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
|
||||
|
||||
zoo3:
|
||||
image: zookeeper
|
||||
image: zookeeper:3.4.12
|
||||
restart: always
|
||||
environment:
|
||||
ZOO_TICK_TIME: 500
|
||||
|
||||
@ -22,6 +22,9 @@ RUN apt-get update \
|
||||
libicu-dev \
|
||||
bsdutils \
|
||||
curl \
|
||||
liblua5.1-dev \
|
||||
luajit \
|
||||
libssl-dev \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
@ -31,7 +34,7 @@ RUN apt-get update \
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN pip install pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis
|
||||
RUN pip install pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 17.09.1-ce
|
||||
|
||||
1
dbms/tests/ints_dictionary.xml
Symbolic link
1
dbms/tests/ints_dictionary.xml
Symbolic link
@ -0,0 +1 @@
|
||||
../../docker/test/stateless/ints_dictionary.xml
|
||||
@ -1,44 +0,0 @@
|
||||
<test>
|
||||
<name>json_extract</name>
|
||||
<type>once</type>
|
||||
|
||||
<stop_conditions>
|
||||
<any_of>
|
||||
<average_speed_not_changing_for_ms>1000</average_speed_not_changing_for_ms>
|
||||
<total_time_ms>2000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<main_metric>
|
||||
<max_rows_per_second />
|
||||
<max_bytes_per_second />
|
||||
<avg_rows_per_second />
|
||||
<avg_bytes_per_second />
|
||||
</main_metric>
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>json</name>
|
||||
<values>
|
||||
<value>'{"sparam":"test_string","nparam": 772}'</value>
|
||||
<value>'{"sparam":{"nested_1":"test_string","nested_2":"test_2"}, "nparam":8495, "fparam":{"nested_1":91.15,"nested_2":[334, 89.05, 1000.01]}, "bparam":false}'</value>
|
||||
</values>
|
||||
</substitution>
|
||||
<substitution>
|
||||
<name>allow_simdjson</name>
|
||||
<values>
|
||||
<value>0</value>
|
||||
<value>1</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractInt(materialize({json}), 'nparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractUInt(materialize({json}), 'nparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({json}), 'fparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractBool(materialize({json}), 'bparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam', 'nested_1')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({json}), 'fparam', 'nested_2', -2)) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
<query>SELECT count() FROM system.numbers WHERE NOT ignore(JSONExtractRaw(materialize({json}), 'fparam')) SETTINGS allow_simdjson={allow_simdjson}</query>
|
||||
</test>
|
||||
52
dbms/tests/performance/json_extract_rapidjson.xml
Normal file
52
dbms/tests/performance/json_extract_rapidjson.xml
Normal file
@ -0,0 +1,52 @@
|
||||
<test>
|
||||
<name>json_extract_rapidjson</name>
|
||||
<type>once</type>
|
||||
|
||||
<stop_conditions>
|
||||
<any_of>
|
||||
<average_speed_not_changing_for_ms>1000</average_speed_not_changing_for_ms>
|
||||
<total_time_ms>2000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<main_metric>
|
||||
<max_rows_per_second />
|
||||
<max_bytes_per_second />
|
||||
<avg_rows_per_second />
|
||||
<avg_bytes_per_second />
|
||||
</main_metric>
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>json</name>
|
||||
<values>
|
||||
<value>'{"sparam":"test_string","nparam": 772}'</value>
|
||||
</values>
|
||||
</substitution>
|
||||
<substitution>
|
||||
<name>long_json</name>
|
||||
<values>
|
||||
<value>'{"sparam":{"nested_1":"test_string","nested_2":"test_2"}, "nparam":8495, "fparam":{"nested_1":91.15,"nested_2":[334, 89.05, 1000.01]}, "bparam":false}'</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<settings>
|
||||
<allow_simdjson>0</allow_simdjson>
|
||||
</settings>
|
||||
|
||||
<query>SELECT 'rapidjson-1', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam'))</query>
|
||||
<query>SELECT 'rapidjson-2', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam', 'nested_1'))</query>
|
||||
<query>SELECT 'rapidjson-3', count() FROM system.numbers WHERE NOT ignore(JSONExtractInt(materialize({json}), 'nparam'))</query>
|
||||
<query>SELECT 'rapidjson-4', count() FROM system.numbers WHERE NOT ignore(JSONExtractUInt(materialize({json}), 'nparam'))</query>
|
||||
<query>SELECT 'rapidjson-5', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({json}), 'fparam'))</query>
|
||||
|
||||
<query>SELECT 'rapidjson-6', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({long_json}), 'sparam'))</query>
|
||||
<query>SELECT 'rapidjson-7', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({long_json}), 'sparam', 'nested_1'))</query>
|
||||
<query>SELECT 'rapidjson-8', count() FROM system.numbers WHERE NOT ignore(JSONExtractInt(materialize({long_json}), 'nparam'))</query>
|
||||
<query>SELECT 'rapidjson-9', count() FROM system.numbers WHERE NOT ignore(JSONExtractUInt(materialize({long_json}), 'nparam'))</query>
|
||||
<query>SELECT 'rapidjson-10', count() FROM system.numbers WHERE NOT ignore(JSONExtractRaw(materialize({long_json}), 'fparam'))</query>
|
||||
<query>SELECT 'rapidjson-11', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({long_json}), 'fparam'))</query>
|
||||
<query>SELECT 'rapidjson-12', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({long_json}), 'fparam', 'nested_2', -2))</query>
|
||||
<query>SELECT 'rapidjson-13', count() FROM system.numbers WHERE NOT ignore(JSONExtractBool(materialize({long_json}), 'bparam'))</query>
|
||||
</test>
|
||||
57
dbms/tests/performance/json_extract_simdjson.xml
Normal file
57
dbms/tests/performance/json_extract_simdjson.xml
Normal file
@ -0,0 +1,57 @@
|
||||
<test>
|
||||
<name>json_extract_simdjson</name>
|
||||
<type>once</type>
|
||||
|
||||
<preconditions>
|
||||
<cpu>AVX2</cpu>
|
||||
</preconditions>
|
||||
|
||||
<stop_conditions>
|
||||
<any_of>
|
||||
<average_speed_not_changing_for_ms>1000</average_speed_not_changing_for_ms>
|
||||
<total_time_ms>2000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<main_metric>
|
||||
<max_rows_per_second />
|
||||
<max_bytes_per_second />
|
||||
<avg_rows_per_second />
|
||||
<avg_bytes_per_second />
|
||||
</main_metric>
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>json</name>
|
||||
<values>
|
||||
<value>'{"sparam":"test_string","nparam": 772}'</value>
|
||||
</values>
|
||||
</substitution>
|
||||
<substitution>
|
||||
<name>long_json</name>
|
||||
<values>
|
||||
<value>'{"sparam":{"nested_1":"test_string","nested_2":"test_2"}, "nparam":8495, "fparam":{"nested_1":91.15,"nested_2":[334, 89.05, 1000.01]}, "bparam":false}'</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<settings>
|
||||
<allow_simdjson>1</allow_simdjson>
|
||||
</settings>
|
||||
|
||||
<query>SELECT 'simdjson-1', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam'))</query>
|
||||
<query>SELECT 'simdjson-2', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({json}), 'sparam', 'nested_1'))</query>
|
||||
<query>SELECT 'simdjson-3', count() FROM system.numbers WHERE NOT ignore(JSONExtractInt(materialize({json}), 'nparam'))</query>
|
||||
<query>SELECT 'simdjson-4', count() FROM system.numbers WHERE NOT ignore(JSONExtractUInt(materialize({json}), 'nparam'))</query>
|
||||
<query>SELECT 'simdjson-5', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({json}), 'fparam'))</query>
|
||||
|
||||
<query>SELECT 'simdjson-6', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({long_json}), 'sparam'))</query>
|
||||
<query>SELECT 'simdjson-7', count() FROM system.numbers WHERE NOT ignore(JSONExtractString(materialize({long_json}), 'sparam', 'nested_1'))</query>
|
||||
<query>SELECT 'simdjson-8', count() FROM system.numbers WHERE NOT ignore(JSONExtractInt(materialize({long_json}), 'nparam'))</query>
|
||||
<query>SELECT 'simdjson-9', count() FROM system.numbers WHERE NOT ignore(JSONExtractUInt(materialize({long_json}), 'nparam'))</query>
|
||||
<query>SELECT 'simdjson-10', count() FROM system.numbers WHERE NOT ignore(JSONExtractRaw(materialize({long_json}), 'fparam'))</query>
|
||||
<query>SELECT 'simdjson-11', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({long_json}), 'fparam'))</query>
|
||||
<query>SELECT 'simdjson-12', count() FROM system.numbers WHERE NOT ignore(JSONExtractFloat(materialize({long_json}), 'fparam', 'nested_2', -2))</query>
|
||||
<query>SELECT 'simdjson-13', count() FROM system.numbers WHERE NOT ignore(JSONExtractBool(materialize({long_json}), 'bparam'))</query>
|
||||
|
||||
</test>
|
||||
37
dbms/tests/performance/linear_regression.xml
Normal file
37
dbms/tests/performance/linear_regression.xml
Normal file
@ -0,0 +1,37 @@
|
||||
<test>
|
||||
<name>linear_regression</name>
|
||||
<type>loop</type>
|
||||
|
||||
<stop_conditions>
|
||||
<any_of>
|
||||
<average_speed_not_changing_for_ms>3000</average_speed_not_changing_for_ms>
|
||||
<total_time_ms>10000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<preconditions>
|
||||
<table_exists>test.hits</table_exists>
|
||||
</preconditions>
|
||||
|
||||
<main_metric>
|
||||
<min_time/>
|
||||
</main_metric>
|
||||
|
||||
<create_query>DROP TABLE IF EXISTS test_model</create_query>
|
||||
<create_query>CREATE TABLE test_model engine = Memory as select stochasticLinearRegressionState(0.0001)(Age, Income, ParamPrice, Robotness, RefererHash) as state from test.hits</create_query>
|
||||
|
||||
<!-- Check model fit-->
|
||||
<query>WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15)(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1</query>
|
||||
<query>SELECT stochasticLinearRegression(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits</query>
|
||||
|
||||
<!-- Check model fit with Momentum-->
|
||||
<query>WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Momentum')(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1</query>
|
||||
|
||||
<!-- Check model fit with Nesterov-->
|
||||
<query>WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Nesterov')(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1</query>
|
||||
|
||||
<!-- Check model predict-->
|
||||
<query>with (SELECT state FROM test_model) as model select evalMLMethod(model, Income, ParamPrice, Robotness, RefererHash) from test.hits</query>
|
||||
|
||||
<drop_query>DROP TABLE IF EXISTS test_model</drop_query>
|
||||
</test>
|
||||
@ -11,7 +11,7 @@
|
||||
</stop_conditions>
|
||||
|
||||
<metrics>
|
||||
<!-- For running one inifinite query -->
|
||||
<!-- For running one infinite query -->
|
||||
<max_rows_per_second />
|
||||
<max_bytes_per_second />
|
||||
<avg_rows_per_second />
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user