Merge branch 'master' into fix-custom-serializations

This commit is contained in:
Anton Popov 2022-07-31 01:51:52 +02:00 committed by GitHub
commit 74eb8119ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
666 changed files with 2106 additions and 2975 deletions

View File

@ -16,5 +16,3 @@ ClickHouse® is an open-source column-oriented database management system that a
## Upcoming events
* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.

View File

@ -16,7 +16,7 @@ option (ENABLE_SSE41 "Use SSE4.1 instructions on x86_64" 1)
option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1)
option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1)
option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
option (ENABLE_AVX "Use AVX instructions on x86_64" 0)
option (ENABLE_AVX "Use AVX instructions on x86_64" 1)
option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)

View File

@ -55,7 +55,7 @@ ccache --zero-stats ||:
if [ "$BUILD_MUSL_KEEPER" == "1" ]
then
# build keeper with musl separately
cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS clickhouse-keeper
@ -70,10 +70,10 @@ then
rm -f CMakeCache.txt
# Build the rest of binaries
cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
else
# Build everything
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
fi
if [ "coverity" == "$COMBINED_OUTPUT" ]
@ -88,7 +88,7 @@ fi
# No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
$SCAN_WRAPPER ninja $NINJA_FLAGS clickhouse-bundle
$SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET
ls -la ./programs

View File

@ -134,6 +134,7 @@ def parse_env_variables(
result = []
result.append("OUTPUT_DIR=/output")
cmake_flags = ["$CMAKE_FLAGS"]
build_target = "clickhouse-bundle"
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
@ -236,7 +237,11 @@ def parse_env_variables(
result.append("CCACHE_BASEDIR=/build")
result.append("CCACHE_NOHASHDIR=true")
result.append("CCACHE_COMPILERCHECK=content")
result.append("CCACHE_MAXSIZE=15G")
cache_maxsize = "15G"
if clang_tidy:
# 15G is not enough for tidy build
cache_maxsize = "25G"
result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
# result.append("CCACHE_UMASK=777")
if distcc_hosts:
@ -268,15 +273,20 @@ def parse_env_variables(
# we have to build them at least in some way in CI. The split build is
# probably the least heavy disk-wise.
cmake_flags.append("-DENABLE_UTILS=1")
# utils are not included into clickhouse-bundle, so build everything
build_target = "all"
if clang_tidy:
cmake_flags.append("-DENABLE_CLANG_TIDY=1")
cmake_flags.append("-DENABLE_UTILS=1")
cmake_flags.append("-DENABLE_TESTS=1")
cmake_flags.append("-DENABLE_EXAMPLES=1")
# Don't stop on first error to find more clang-tidy errors in one run.
result.append("NINJA_FLAGS=-k0")
cmake_flags.append("-DENABLE_UTILS=1")
# utils are not included into clickhouse-bundle, so build everything
build_target = "all"
if with_coverage:
cmake_flags.append("-DWITH_COVERAGE=1")
@ -290,6 +300,7 @@ def parse_env_variables(
cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')
result.append(f"BUILD_TARGET={build_target}")
return result

View File

@ -34,13 +34,14 @@ RUN apt-get update \
ARG TARGETARCH
# Install MySQL ODBC driver from RHEL rpm
# For reference https://downloads.mysql.com/archives/c-odbc/ RHEL
RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) rarch=x86_64 ;; \
arm64) rarch=aarch64 ;; \
esac \
&& cd /tmp \
&& curl -o mysql-odbc.rpm "https://cdn.mysql.com/Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \
&& curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \
&& rpm2archive mysql-odbc.rpm \
&& tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \
&& LINK_DIR=$(dpkg -L libodbc1 | grep '^/usr/lib/.*-linux-gnu/odbc$') \

View File

@ -149,7 +149,7 @@ Features:
### ClickCat {#clickcat}
[ClickCat](https://github.com/clickcat-project/ClickCat) is a firendly user interface that lets you search, explore and visualize your ClickHouse Data.
[ClickCat](https://github.com/clickcat-project/ClickCat) is a friendly user interface that lets you search, explore and visualize your ClickHouse Data.
Features:

View File

@ -5,12 +5,12 @@ sidebar_label: Testing Hardware
# How to Test Your Hardware with ClickHouse
You can run basic ClickHouse performance test on any server without installation of ClickHouse packages.
You can run a basic ClickHouse performance test on any server without installation of ClickHouse packages.
## Automated Run
You can run benchmark with a single script.
You can run the benchmark with a single script.
1. Download the script.
```
@ -26,58 +26,3 @@ chmod a+x ./hardware.sh
3. Copy the output and send it to feedback@clickhouse.com
All the results are published here: https://clickhouse.com/benchmark/hardware/
## Manual Run
Alternatively you can perform benchmark in the following steps.
1. ssh to the server and download the binary with wget:
```bash
# For amd64:
wget https://builds.clickhouse.com/master/amd64/clickhouse
# For aarch64:
wget https://builds.clickhouse.com/master/aarch64/clickhouse
# For powerpc64le:
wget https://builds.clickhouse.com/master/powerpc64le/clickhouse
# For freebsd:
wget https://builds.clickhouse.com/master/freebsd/clickhouse
# For freebsd-aarch64:
wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse
# For freebsd-powerpc64le:
wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse
# For macos:
wget https://builds.clickhouse.com/master/macos/clickhouse
# For macos-aarch64:
wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse
# Then do:
chmod a+x clickhouse
```
2. Download benchmark files:
```bash
wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/benchmark-new.sh
chmod a+x benchmark-new.sh
wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/queries.sql
```
3. Download the [web analytics dataset](../getting-started/example-datasets/metrica.md) (“hits” table containing 100 million rows).
```bash
wget https://datasets.clickhouse.com/hits/partitions/hits_100m_obfuscated_v1.tar.xz
tar xvf hits_100m_obfuscated_v1.tar.xz -C .
mv hits_100m_obfuscated_v1/* .
```
4. Run the server:
```bash
./clickhouse server
```
5. Check the data: ssh to the server in another terminal
```bash
./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated"
100000000
```
6. Run the benchmark:
```bash
./benchmark-new.sh hits_100m_obfuscated
```
7. Send the numbers and the info about your hardware configuration to feedback@clickhouse.com
All the results are published here: https://clickhouse.com/benchmark/hardware/

View File

@ -956,9 +956,28 @@ SELECT
## timeSlots(StartTime, Duration,\[, Size\])
For a time interval starting at StartTime and continuing for Duration seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the Size in seconds. Size is an optional parameter: a constant UInt32, set to 1800 by default.
For example, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`.
This is necessary for searching for pageviews in the corresponding session.
For a time interval starting at StartTime and continuing for Duration seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the Size in seconds. Size is an optional parameter set to 1800 (30 minutes) by default.
This is necessary, for example, when searching for pageviews in the corresponding session.
Accepts DateTime and DateTime64 as StartTime argument. For DateTime, Duration and Size arguments must be `UInt32`. For DateTime64 they must be `Decimal64`.
Returns an array of DateTime/DateTime64 (return type matches the type of StartTime). For DateTime64, the return value's scale can differ from the scale of StartTime --- the highest scale among all given arguments is taken.
Example:
```sql
SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
```
``` text
┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │
└─────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐
│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │
└─────────────────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐
│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## formatDateTime

View File

@ -944,14 +944,31 @@ SELECT now('Europe/Moscow');
## timeSlot {#timeslot}
Округляет время до получаса.
Эта функция является специфичной для Яндекс.Метрики, так как пол часа - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит.
Эта функция является специфичной для Яндекс.Метрики, так как полчаса - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит.
## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size}
Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр.
Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию.
Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`.
Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра StartTime). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента StartTime --- результат будет иметь наибольший масштаб среди всех данных аргументов.
Для интервала времени, начинающегося в StartTime и продолжающегося Duration секунд, возвращает массив моментов времени, состоящий из округлений вниз до Size точек в секундах из этого интервала. Size - необязательный параметр, константный UInt32, по умолчанию равен 1800.
Например, `timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`.
Это нужно для поиска хитов, входящих в соответствующий визит.
Пример использования:
```sql
SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
```
``` text
┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │
└─────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐
│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │
└─────────────────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐
│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## toYYYYMM

View File

@ -62,6 +62,8 @@ ORDER BY expr
- `PARTITION BY` — [分区键](custom-partitioning-key.md) ,可选项。
大多数情况下,不需要分使用区键。即使需要使用,也不需要使用比月更细粒度的分区键。分区不会加快查询(这与 ORDER BY 表达式不同)。永远也别使用过细粒度的分区键。不要使用客户端指定分区标识符或分区字段名称来对数据进行分区(而是将分区字段标识或名称作为 ORDER BY 表达式的第一列来指定分区)。
要按月分区,可以使用表达式 `toYYYYMM(date_column)` ,这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"`
- `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key),在这里指定,可选项。

View File

@ -195,6 +195,12 @@ else()
message(STATUS "ClickHouse disks mode: OFF")
endif()
if (ENABLE_CLICKHOUSE_SU)
message(STATUS "ClickHouse su: ON")
else()
message(STATUS "ClickHouse su: OFF")
endif()
configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h)
macro(clickhouse_target_link_split_lib target name)
@ -367,6 +373,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
endif ()
if (ENABLE_CLICKHOUSE_SU)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-su)
endif ()
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})

View File

@ -1,3 +1,3 @@
set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp)
set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp su.cpp)
set (CLICKHOUSE_SU_LINK PRIVATE dbms)
clickhouse_program_add(su)

View File

@ -1,145 +1,2 @@
#include <Common/Exception.h>
#include <IO/ReadHelpers.h>
#include <fmt/format.h>
#include <vector>
#include <sys/types.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
/// "su" means "set user"
/// In fact, this program can set Unix user and group.
///
/// Usage:
/// clickhouse su user[:group] args...
///
/// - will set user and, optionally, group and exec the remaining args.
/// user and group can be numeric identifiers or strings.
///
/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker.
/// People want to run programs inside Docker with dropped privileges (less than root).
/// But the standard Linux "su" program is not suitable for usage inside Docker,
/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security,
/// but Docker is also doing something with the terminal and it is incompatible.
/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec".
/// But it would be very strange to use 3rd-party software only to do two-three syscalls.
/// That's why we provide this tool.
///
/// Note: ClickHouse does not need Docker at all and works better without Docker.
/// ClickHouse has no dependencies, it is packaged and distributed in single binary.
/// There is no reason to use Docker unless you are already running all your software in Docker.
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int SYSTEM_ERROR;
}
void setUserAndGroup(std::string arg_uid, std::string arg_gid)
{
static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw.
std::unique_ptr<char[]> buf(new char[buf_size]);
/// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later.
if (!arg_gid.empty())
{
gid_t gid = 0;
if (!tryParse(gid, arg_gid) || gid == 0)
{
group entry{};
group * result{};
if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
gid = entry.gr_gid;
}
if (gid == 0 && getgid() != 0)
throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setgid(gid))
throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
}
if (!arg_uid.empty())
{
/// Is it numeric id or name?
uid_t uid = 0;
if (!tryParse(uid, arg_uid) || uid == 0)
{
passwd entry{};
passwd * result{};
if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
uid = entry.pw_uid;
}
if (uid == 0 && getuid() != 0)
throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setuid(uid))
throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
}
}
}
int mainEntryClickHouseSU(int argc, char ** argv)
try
{
using namespace DB;
if (argc < 3)
{
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
exit(0);
}
std::string_view user_and_group = argv[1];
std::string user;
std::string group;
auto pos = user_and_group.find(':');
if (pos == std::string_view::npos)
{
user = user_and_group;
}
else
{
user = user_and_group.substr(0, pos);
group = user_and_group.substr(pos + 1);
}
setUserAndGroup(std::move(user), std::move(group));
std::vector<char *> new_argv;
new_argv.reserve(argc - 1);
new_argv.insert(new_argv.begin(), argv + 2, argv + argc);
new_argv.push_back(nullptr);
execvp(new_argv.front(), new_argv.data());
throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR);
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(false) << '\n';
return 1;
}
int mainEntryClickHouseSU(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); }

145
programs/su/su.cpp Normal file
View File

@ -0,0 +1,145 @@
#include <Common/Exception.h>
#include <IO/ReadHelpers.h>
#include <fmt/format.h>
#include <vector>
#include <sys/types.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
/// "su" means "set user"
/// In fact, this program can set Unix user and group.
///
/// Usage:
/// clickhouse su user[:group] args...
///
/// - will set user and, optionally, group and exec the remaining args.
/// user and group can be numeric identifiers or strings.
///
/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker.
/// People want to run programs inside Docker with dropped privileges (less than root).
/// But the standard Linux "su" program is not suitable for usage inside Docker,
/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security,
/// but Docker is also doing something with the terminal and it is incompatible.
/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec".
/// But it would be very strange to use 3rd-party software only to do two-three syscalls.
/// That's why we provide this tool.
///
/// Note: ClickHouse does not need Docker at all and works better without Docker.
/// ClickHouse has no dependencies, it is packaged and distributed in single binary.
/// There is no reason to use Docker unless you are already running all your software in Docker.
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int SYSTEM_ERROR;
}
void setUserAndGroup(std::string arg_uid, std::string arg_gid)
{
static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw.
std::unique_ptr<char[]> buf(new char[buf_size]);
/// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later.
if (!arg_gid.empty())
{
gid_t gid = 0;
if (!tryParse(gid, arg_gid) || gid == 0)
{
group entry{};
group * result{};
if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
gid = entry.gr_gid;
}
if (gid == 0 && getgid() != 0)
throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setgid(gid))
throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
}
if (!arg_uid.empty())
{
/// Is it numeric id or name?
uid_t uid = 0;
if (!tryParse(uid, arg_uid) || uid == 0)
{
passwd entry{};
passwd * result{};
if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
uid = entry.pw_uid;
}
if (uid == 0 && getuid() != 0)
throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setuid(uid))
throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
}
}
}
int mainEntryClickHouseSU(int argc, char ** argv)
try
{
using namespace DB;
if (argc < 3)
{
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
exit(0);
}
std::string_view user_and_group = argv[1];
std::string user;
std::string group;
auto pos = user_and_group.find(':');
if (pos == std::string_view::npos)
{
user = user_and_group;
}
else
{
user = user_and_group.substr(0, pos);
group = user_and_group.substr(pos + 1);
}
setUserAndGroup(std::move(user), std::move(group));
std::vector<char *> new_argv;
new_argv.reserve(argc - 1);
new_argv.insert(new_argv.begin(), argv + 2, argv + argc);
new_argv.push_back(nullptr);
execvp(new_argv.front(), new_argv.data());
throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR);
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(false) << '\n';
return 1;
}

View File

@ -25,7 +25,7 @@ int main(int argc, char ** argv)
return 0;
}
String cache_name = "";
const char * cache_name = "";
if (argc == 4)
cache_name = argv[3];

View File

@ -144,8 +144,8 @@ endif ()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp)
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h)
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp)
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h)
list (APPEND dbms_sources
AggregateFunctions/IAggregateFunction.cpp

View File

@ -130,7 +130,7 @@ public:
private:
ReadBuffer & in;
/// The physical location of the current cell.
Locus locus;
Locus locus{};
/// The current position in the file as a cell number.
BucketIndex current_bucket_index = 0;
/// The number of bytes read.

View File

@ -0,0 +1,12 @@
#include <Common/register_objects.h>
namespace DB
{
FunctionRegisterMap & FunctionRegisterMap::instance()
{
static FunctionRegisterMap map;
return map;
}
}

View File

@ -0,0 +1,33 @@
#pragma once
#include <string_view>
#include <unordered_map>
namespace DB
{
class FunctionFactory;
using FunctionRegisterFunctionPtr = void (*)(::DB::FunctionFactory &);
struct FunctionRegisterMap : public std::unordered_map<std::string_view, FunctionRegisterFunctionPtr>
{
static FunctionRegisterMap & instance();
};
struct FunctionRegister
{
FunctionRegister(std::string_view name, FunctionRegisterFunctionPtr func_ptr)
{
FunctionRegisterMap::instance().emplace(std::move(name), func_ptr);
}
};
}
#define REGISTER_FUNCTION_IMPL(fn, func_name, register_name) \
void func_name(::DB::FunctionFactory & factory); \
static ::DB::FunctionRegister register_name(#fn, func_name); \
void func_name(::DB::FunctionFactory & factory)
#define REGISTER_FUNCTION(fn) REGISTER_FUNCTION_IMPL(fn, registerFunction##fn, REGISTER_FUNCTION_##fn)

79
src/Core/Joins.cpp Normal file
View File

@ -0,0 +1,79 @@
#include <Core/Joins.h>
namespace DB
{
const char * toString(JoinKind kind)
{
switch (kind)
{
case JoinKind::Inner: return "INNER";
case JoinKind::Left: return "LEFT";
case JoinKind::Right: return "RIGHT";
case JoinKind::Full: return "FULL";
case JoinKind::Cross: return "CROSS";
case JoinKind::Comma: return "COMMA";
}
};
const char * toString(JoinStrictness strictness)
{
switch (strictness)
{
case JoinStrictness::Unspecified: return "UNSPECIFIED";
case JoinStrictness::RightAny: return "RIGHT_ANY";
case JoinStrictness::Any: return "ANY";
case JoinStrictness::All: return "ALL";
case JoinStrictness::Asof: return "ASOF";
case JoinStrictness::Semi: return "SEMI";
case JoinStrictness::Anti: return "ANTI";
}
}
const char * toString(JoinLocality locality)
{
switch (locality)
{
case JoinLocality::Unspecified: return "UNSPECIFIED";
case JoinLocality::Local: return "LOCAL";
case JoinLocality::Global: return "GLOBAL";
}
}
const char * toString(ASOFJoinInequality asof_join_inequality)
{
switch (asof_join_inequality)
{
case ASOFJoinInequality::None: return "NONE";
case ASOFJoinInequality::Less: return "LESS";
case ASOFJoinInequality::Greater: return "GREATER";
case ASOFJoinInequality::LessOrEquals: return "LESS_OR_EQUALS";
case ASOFJoinInequality::GreaterOrEquals: return "GREATER_OR_EQUALS";
}
}
const char * toString(JoinAlgorithm join_algorithm)
{
switch (join_algorithm)
{
case JoinAlgorithm::DEFAULT: return "DEFAULT";
case JoinAlgorithm::AUTO: return "AUTO";
case JoinAlgorithm::HASH: return "HASH";
case JoinAlgorithm::PARTIAL_MERGE: return "PARTIAL_MERGE";
case JoinAlgorithm::PREFER_PARTIAL_MERGE: return "PREFER_PARTIAL_MERGE";
case JoinAlgorithm::PARALLEL_HASH: return "PARALLEL_HASH";
case JoinAlgorithm::DIRECT: return "DIRECT";
case JoinAlgorithm::FULL_SORTING_MERGE: return "FULL_SORTING_MERGE";
}
}
const char * toString(JoinTableSide join_table_side)
{
switch (join_table_side)
{
case JoinTableSide::Left: return "LEFT";
case JoinTableSide::Right: return "RIGHT";
}
}
}

119
src/Core/Joins.h Normal file
View File

@ -0,0 +1,119 @@
#pragma once
#include <string_view>
namespace DB
{
/// Join method.
enum class JoinKind
{
Inner, /// Leave only rows that was JOINed.
Left, /// If in "right" table there is no corresponding rows, use default values instead.
Right,
Full,
Cross, /// Direct product. Strictness and condition doesn't matter.
Comma /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE.
};
const char * toString(JoinKind kind);
inline constexpr bool isLeft(JoinKind kind) { return kind == JoinKind::Left; }
inline constexpr bool isRight(JoinKind kind) { return kind == JoinKind::Right; }
inline constexpr bool isInner(JoinKind kind) { return kind == JoinKind::Inner; }
inline constexpr bool isFull(JoinKind kind) { return kind == JoinKind::Full; }
inline constexpr bool isCrossOrComma(JoinKind kind) { return kind == JoinKind::Comma || kind == JoinKind::Cross; }
inline constexpr bool isRightOrFull(JoinKind kind) { return kind == JoinKind::Right || kind == JoinKind::Full; }
inline constexpr bool isLeftOrFull(JoinKind kind) { return kind == JoinKind::Left || kind == JoinKind::Full; }
inline constexpr bool isInnerOrRight(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Right; }
inline constexpr bool isInnerOrLeft(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Left; }
/// Allows more optimal JOIN for typical cases.
enum class JoinStrictness
{
Unspecified,
RightAny, /// Old ANY JOIN. If there are many suitable rows in right table, use any from them to join.
Any, /// Semi Join with any value from filtering table. For LEFT JOIN with Any and RightAny are the same.
All, /// If there are many suitable rows to join, use all of them and replicate rows of "left" table (usual semantic of JOIN).
Asof, /// For the last JOIN column, pick the latest value
Semi, /// LEFT or RIGHT. SEMI LEFT JOIN filters left table by values exists in right table. SEMI RIGHT - otherwise.
Anti, /// LEFT or RIGHT. Same as SEMI JOIN but filter values that are NOT exists in other table.
};
const char * toString(JoinStrictness strictness);
/// Algorithm for distributed query processing.
enum class JoinLocality
{
Unspecified,
Local, /// Perform JOIN, using only data available on same servers (co-located data).
Global /// Collect and merge data from remote servers, and broadcast it to each server.
};
const char * toString(JoinLocality locality);
/// ASOF JOIN inequality type
enum class ASOFJoinInequality
{
None,
Less,
Greater,
LessOrEquals,
GreaterOrEquals,
};
const char * toString(ASOFJoinInequality asof_join_inequality);
inline constexpr ASOFJoinInequality getASOFJoinInequality(std::string_view func_name)
{
ASOFJoinInequality inequality = ASOFJoinInequality::None;
if (func_name == "less")
inequality = ASOFJoinInequality::Less;
else if (func_name == "greater")
inequality = ASOFJoinInequality::Greater;
else if (func_name == "lessOrEquals")
inequality = ASOFJoinInequality::LessOrEquals;
else if (func_name == "greaterOrEquals")
inequality = ASOFJoinInequality::GreaterOrEquals;
return inequality;
}
inline constexpr ASOFJoinInequality reverseASOFJoinInequality(ASOFJoinInequality inequality)
{
if (inequality == ASOFJoinInequality::Less)
return ASOFJoinInequality::Greater;
else if (inequality == ASOFJoinInequality::Greater)
return ASOFJoinInequality::Less;
else if (inequality == ASOFJoinInequality::LessOrEquals)
return ASOFJoinInequality::GreaterOrEquals;
else if (inequality == ASOFJoinInequality::GreaterOrEquals)
return ASOFJoinInequality::LessOrEquals;
return ASOFJoinInequality::None;
}
enum class JoinAlgorithm
{
DEFAULT = 0,
AUTO,
HASH,
PARTIAL_MERGE,
PREFER_PARTIAL_MERGE,
PARALLEL_HASH,
DIRECT,
FULL_SORTING_MERGE,
};
const char * toString(JoinAlgorithm join_algorithm);
enum class JoinTableSide
{
Left,
Right
};
const char * toString(JoinTableSide join_table_side);
}

View File

@ -247,7 +247,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
\
M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \
\
M(JoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \
M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \
M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \
\
M(UInt64, preferred_block_size_bytes, 1000000, "", 0) \

View File

@ -26,8 +26,8 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING,
IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN,
{{"", JoinStrictness::Unspecified},
{"ALL", JoinStrictness::ALL},
{"ANY", JoinStrictness::ANY}})
{"ALL", JoinStrictness::All},
{"ANY", JoinStrictness::Any}})
IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN,

View File

@ -1,12 +1,14 @@
#pragma once
#include <Core/SettingsFields.h>
#include <Core/Joins.h>
#include <QueryPipeline/SizeLimits.h>
#include <Formats/FormatSettings.h>
namespace DB
{
enum class LoadBalancing
{
/// among replicas with a minimum number of errors selected randomly
@ -26,28 +28,8 @@ enum class LoadBalancing
DECLARE_SETTING_ENUM(LoadBalancing)
enum class JoinStrictness
{
Unspecified = 0, /// Query JOIN without strictness will throw Exception.
ALL, /// Query JOIN without strictness -> ALL JOIN ...
ANY, /// Query JOIN without strictness -> ANY JOIN ...
};
DECLARE_SETTING_ENUM(JoinStrictness)
enum class JoinAlgorithm
{
DEFAULT = 0,
AUTO,
HASH,
PARTIAL_MERGE,
PREFER_PARTIAL_MERGE,
PARALLEL_HASH,
DIRECT,
FULL_SORTING_MERGE,
};
DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm)
@ -200,4 +182,5 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EnumComparingMode, FormatSettings::EnumComparin
DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule)
DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation)
}

View File

@ -12,7 +12,7 @@
#if defined(__clang__)
#include <experimental/coroutine>
namespace std
namespace std // NOLINT(cert-dcl58-cpp)
{
using namespace experimental::coroutines_v1;
}
@ -97,14 +97,14 @@ struct Task
static bool resumeImpl(Task *r)
{
if (r->value)
return false;
return false;
auto & next = r->my.promise().next;
if (next)
{
if (resumeImpl(next.promise().r))
return true;
return true;
next = {};
}
@ -183,7 +183,7 @@ int main()
auto t = foo("foo");
std::cout << ".. started" << std::endl;
while (t.resume())
std::cout << ".. yielded" << std::endl;
std::cout << ".. yielded" << std::endl;
std::cout << ".. done: " << t.res() << std::endl;
}
catch (DB::Exception & e)

View File

@ -171,13 +171,13 @@ ColumnUInt8::Ptr DirectDictionary<dictionary_key_type>::hasKeys(
auto requested_keys = requested_keys_extractor.extractAllKeys();
size_t requested_keys_size = requested_keys.size();
HashMap<KeyType, size_t> requested_key_to_index;
HashMap<KeyType, PaddedPODArray<size_t>> requested_key_to_index;
requested_key_to_index.reserve(requested_keys_size);
for (size_t i = 0; i < requested_keys.size(); ++i)
{
auto requested_key = requested_keys[i];
requested_key_to_index[requested_key] = i;
requested_key_to_index[requested_key].push_back(i);
}
auto result = ColumnUInt8::create(requested_keys_size, false);
@ -208,10 +208,13 @@ ColumnUInt8::Ptr DirectDictionary<dictionary_key_type>::hasKeys(
const auto * it = requested_key_to_index.find(block_key);
assert(it);
size_t result_data_found_index = it->getMapped();
/// block_keys_size cannot be used, due to duplicates.
keys_found += !result_data[result_data_found_index];
result_data[result_data_found_index] = true;
auto & result_data_found_indexes = it->getMapped();
for (size_t result_data_found_index : result_data_found_indexes)
{
/// block_keys_size cannot be used, due to duplicates.
keys_found += !result_data[result_data_found_index];
result_data[result_data_found_index] = true;
}
block_keys_extractor.rollbackCurrentKey();
}

View File

@ -6,13 +6,14 @@ add_subdirectory(divide)
include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
add_headers_and_sources(clickhouse_functions .)
list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp)
list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h)
list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp extractTimeZoneFromFunctionArguments.cpp FunctionsLogical.cpp)
list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h extractTimeZoneFromFunctionArguments.h FunctionsLogical.h)
add_library(clickhouse_functions ${clickhouse_functions_sources})
add_library(clickhouse_functions_obj OBJECT ${clickhouse_functions_sources})
target_link_libraries(clickhouse_functions
PUBLIC
list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_obj>)
list (APPEND PUBLIC_LIBS
ch_contrib::wyhash
ch_contrib::cityhash
ch_contrib::farmhash
@ -24,27 +25,28 @@ target_link_libraries(clickhouse_functions
ch_contrib::metrohash
ch_contrib::murmurhash
ch_contrib::hashidsxx
)
PRIVATE
list (APPEND PRIVATE_LIBS
ch_contrib::zlib
boost::filesystem
divide_impl
)
if (TARGET OpenSSL::Crypto)
target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto)
list (APPEND PUBLIC_LIBS OpenSSL::Crypto)
endif()
if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options(clickhouse_functions PRIVATE "-g0")
target_compile_options(clickhouse_functions_obj PRIVATE "-g0")
endif()
if (TARGET ch_contrib::icu)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::icu)
list (APPEND PRIVATE_LIBS ch_contrib::icu)
endif ()
if (TARGET ch_contrib::fastops)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::fastops)
list (APPEND PRIVATE_LIBS ch_contrib::fastops)
endif ()
if (ENABLE_EXAMPLES)
@ -52,45 +54,46 @@ if (ENABLE_EXAMPLES)
endif ()
if (TARGET ch_contrib::llvm)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm)
list (APPEND PRIVATE_LIBS ch_contrib::llvm)
endif ()
if (TARGET ch_contrib::base64)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64)
list (APPEND PRIVATE_LIBS ch_contrib::base64)
endif()
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4)
list (APPEND PRIVATE_LIBS ch_contrib::lz4)
if (ENABLE_NLP)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::cld2)
list (APPEND PRIVATE_LIBS ch_contrib::cld2)
endif()
if (TARGET ch_contrib::h3)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3)
list (APPEND PRIVATE_LIBS ch_contrib::h3)
endif()
if (TARGET ch_contrib::vectorscan)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::vectorscan)
list (APPEND PRIVATE_LIBS ch_contrib::vectorscan)
endif()
if (TARGET ch_contrib::simdjson)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson)
list (APPEND PRIVATE_LIBS ch_contrib::simdjson)
endif()
if (TARGET ch_contrib::rapidjson)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::rapidjson)
list (APPEND PRIVATE_LIBS ch_contrib::rapidjson)
endif()
add_subdirectory(GatherUtils)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_gatherutils)
list (APPEND PRIVATE_LIBS clickhouse_functions_gatherutils)
add_subdirectory(URL)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_url)
list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_url>)
add_subdirectory(array)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array)
list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_array>)
add_subdirectory(JSONPath)
list (APPEND PRIVATE_LIBS clickhouse_functions_jsonpath)
# Signed integer overflow on user-provided data inside boost::geometry - ignore.
set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow)
@ -98,3 +101,15 @@ set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-s
if (ENABLE_FUZZING)
add_compile_definitions(FUZZING_MODE=1)
endif ()
target_link_libraries(clickhouse_functions_obj PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS})
if (USE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
# Used to forward the linking information to the final binaries such as clickhouse / unit_tests_dbms,
# since such information are lost after we convert to OBJECT target
add_library(clickhouse_functions INTERFACE)
target_link_libraries(clickhouse_functions INTERFACE ${OBJECT_LIBS} ${PUBLIC_LIBS} ${PRIVATE_LIBS})
else()
add_library(clickhouse_functions SHARED ${OBJECT_LIBS})
target_link_libraries(clickhouse_functions PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS})
endif ()

View File

@ -143,7 +143,7 @@ void registerFunctionCRCImpl(FunctionFactory & factory)
factory.registerFunction<T>(T::name, FunctionFactory::CaseInsensitive);
}
void registerFunctionCRC(FunctionFactory & factory)
REGISTER_FUNCTION(CRC)
{
registerFunctionCRCImpl<FunctionCRC32ZLIB>(factory);
registerFunctionCRCImpl<FunctionCRC32IEEE>(factory);

View File

@ -5,7 +5,7 @@
namespace DB
{
void registerCastOverloadResolvers(FunctionFactory & factory)
REGISTER_FUNCTION(CastOverloadResolvers)
{
factory.registerFunction<CastInternalOverloadResolver<CastType::nonAccurate>>(FunctionFactory::CaseInsensitive);
factory.registerFunction<CastInternalOverloadResolver<CastType::accurate>>();

View File

@ -113,7 +113,7 @@ private:
}
};
void registerFunctionChar(FunctionFactory & factory)
REGISTER_FUNCTION(Char)
{
factory.registerFunction<FunctionChar>(FunctionFactory::CaseInsensitive);
}

View File

@ -44,7 +44,7 @@ public:
};
void registerFunctionFQDN(FunctionFactory & factory)
REGISTER_FUNCTION(FQDN)
{
factory.registerFunction<FunctionFQDN>(FunctionFactory::CaseInsensitive);
factory.registerFunction<FunctionFQDN>("fullHostName");

View File

@ -1,6 +1,7 @@
#pragma once
#include <Interpreters/Context_fwd.h>
#include <Common/register_objects.h>
#include <Common/IFactoryWithAliases.h>
#include <Functions/IFunction.h>
#include <Functions/IFunctionAdaptors.h>

View File

@ -94,7 +94,7 @@ public:
};
void registerFunctionFile(FunctionFactory & factory)
REGISTER_FUNCTION(File)
{
factory.registerFunction<FunctionFile>();
}

View File

@ -4,7 +4,7 @@
namespace DB
{
void registerFunctionHashID(FunctionFactory & factory)
REGISTER_FUNCTION(HashID)
{
factory.registerFunction<FunctionHashID>();
}

View File

@ -96,7 +96,7 @@ FunctionBasePtr JoinGetOverloadResolver<or_null>::buildImpl(const ColumnsWithTyp
return std::make_unique<FunctionJoinGet<or_null>>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type);
}
void registerFunctionJoinGet(FunctionFactory & factory)
REGISTER_FUNCTION(JoinGet)
{
// joinGet
factory.registerFunction<JoinGetOverloadResolver<false>>();

View File

@ -5,7 +5,7 @@
namespace DB
{
void registerFunctionsSQLJSON(FunctionFactory & factory)
REGISTER_FUNCTION(SQLJSON)
{
factory.registerFunction<FunctionSQLJSON<NameJSONExists, JSONExistsImpl>>();
factory.registerFunction<FunctionSQLJSON<NameJSONQuery, JSONQueryImpl>>();

View File

@ -4,7 +4,7 @@
namespace DB
{
void registerFunctionShowCertificate(FunctionFactory & factory)
REGISTER_FUNCTION(ShowCertificate)
{
factory.registerFunction<FunctionShowCertificate>();
}

View File

@ -3,12 +3,12 @@
namespace DB
{
void registerFunctionBase58Encode(FunctionFactory & factory)
REGISTER_FUNCTION(Base58Encode)
{
factory.registerFunction<FunctionBase58Conversion<Base58Encode>>();
}
void registerFunctionBase58Decode(FunctionFactory & factory)
REGISTER_FUNCTION(Base58Decode)
{
factory.registerFunction<FunctionBase58Conversion<Base58Decode>>();
}

View File

@ -621,7 +621,7 @@ public:
}
};
void registerFunctionsBinaryRepr(FunctionFactory & factory)
REGISTER_FUNCTION(BinaryRepr)
{
factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>(FunctionFactory::CaseInsensitive);
factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>(FunctionFactory::CaseInsensitive);

View File

@ -329,7 +329,7 @@ public:
}
void registerFunctionsBitToArray(FunctionFactory & factory)
REGISTER_FUNCTION(BitToArray)
{
factory.registerFunction<FunctionBitPositionsToArray>();
factory.registerFunction<FunctionBitmaskToArray>();

View File

@ -7,7 +7,7 @@
namespace DB
{
void registerFunctionsBitmap(FunctionFactory & factory)
REGISTER_FUNCTION(Bitmap)
{
factory.registerFunction<FunctionBitmapBuild>();
factory.registerFunction<FunctionBitmapToArray>();

View File

@ -143,7 +143,7 @@ struct NameDetectLanguageUnknown
using FunctionDetectCharset = FunctionTextClassificationString<CharsetClassificationImpl<false>, NameDetectCharset>;
using FunctionDetectLanguageUnknown = FunctionTextClassificationString<CharsetClassificationImpl<true>, NameDetectLanguageUnknown>;
void registerFunctionDetectCharset(FunctionFactory & factory)
REGISTER_FUNCTION(DetectCharset)
{
factory.registerFunction<FunctionDetectCharset>();
factory.registerFunction<FunctionDetectLanguageUnknown>();

View File

@ -1128,7 +1128,7 @@ public:
struct NameFunctionIPv4NumToString { static constexpr auto name = "IPv4NumToString"; };
struct NameFunctionIPv4NumToStringClassC { static constexpr auto name = "IPv4NumToStringClassC"; };
void registerFunctionsCoding(FunctionFactory & factory)
REGISTER_FUNCTION(Coding)
{
factory.registerFunction<FunctionCutIPv6>();
factory.registerFunction<FunctionIPv4ToIPv6>();

View File

@ -229,7 +229,7 @@ public:
}
};
void registerFunctionsCodingUUID(FunctionFactory & factory)
REGISTER_FUNCTION(CodingUUID)
{
factory.registerFunction<FunctionUUIDNumToString>();
factory.registerFunction<FunctionUUIDStringToNum>();

View File

@ -5,11 +5,7 @@
namespace DB
{
void registerFunctionFixedString(FunctionFactory & factory);
void registerCastOverloadResolvers(FunctionFactory & factory);
void registerFunctionsConversion(FunctionFactory & factory)
REGISTER_FUNCTION(Conversion)
{
factory.registerFunction<FunctionToUInt8>();
factory.registerFunction<FunctionToUInt16>();
@ -41,12 +37,8 @@ void registerFunctionsConversion(FunctionFactory & factory)
factory.registerFunction<FunctionToUUID>();
factory.registerFunction<FunctionToString>();
registerFunctionFixedString(factory);
factory.registerFunction<FunctionToUnixTimestamp>();
registerCastOverloadResolvers(factory);
factory.registerFunction<FunctionToUInt8OrZero>();
factory.registerFunction<FunctionToUInt16OrZero>();
factory.registerFunction<FunctionToUInt32OrZero>();

View File

@ -191,27 +191,29 @@ struct ConvertImpl
vec_null_map_to = &col_null_map_to->getData();
}
bool result_is_bool = isBool(result_type);
for (size_t i = 0; i < input_rows_count; ++i)
if constexpr (std::is_same_v<ToDataType, DataTypeUInt8>)
{
if constexpr (std::is_same_v<ToDataType, DataTypeUInt8>)
if (isBool(result_type))
{
if (result_is_bool)
for (size_t i = 0; i < input_rows_count; ++i)
{
vec_to[i] = vec_from[i] != FromFieldType(0);
continue;
}
goto done;
}
}
if constexpr (std::is_same_v<FromDataType, DataTypeUUID> != std::is_same_v<ToDataType, DataTypeUUID>)
if constexpr (std::is_same_v<FromDataType, DataTypeUUID> != std::is_same_v<ToDataType, DataTypeUUID>)
{
throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED);
}
else
{
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
{
throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED);
}
else
{
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
{
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
for (size_t i = 0; i < input_rows_count; ++i)
{
ToFieldType result;
bool convert_result = false;
@ -231,7 +233,10 @@ struct ConvertImpl
(*vec_null_map_to)[i] = true;
}
}
else
}
else
{
for (size_t i = 0; i < input_rows_count; ++i)
{
if constexpr (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
vec_to[i] = convertDecimals<FromDataType, ToDataType>(vec_from[i], col_from->getScale(), col_to->getScale());
@ -243,10 +248,13 @@ struct ConvertImpl
throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE);
}
}
else
}
else
{
/// If From Data is Nan or Inf and we convert to integer type, throw exception
if constexpr (std::is_floating_point_v<FromFieldType> && !std::is_floating_point_v<ToFieldType>)
{
/// If From Data is Nan or Inf and we convert to integer type, throw exception
if constexpr (std::is_floating_point_v<FromFieldType> && !std::is_floating_point_v<ToFieldType>)
for (size_t i = 0; i < input_rows_count; ++i)
{
if (!isFinite(vec_from[i]))
{
@ -254,15 +262,46 @@ struct ConvertImpl
{
vec_to[i] = 0;
(*vec_null_map_to)[i] = true;
continue;
}
else
throw Exception("Unexpected inf or nan to integer conversion", ErrorCodes::CANNOT_CONVERT_TYPE);
}
}
else
{
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>)
{
bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]);
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>)
if (!convert_result)
{
if (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
{
vec_to[i] = 0;
(*vec_null_map_to)[i] = true;
}
else
{
throw Exception(
"Value in column " + named_from.column->getName() + " cannot be safely converted into type "
+ result_type->getName(),
ErrorCodes::CANNOT_CONVERT_TYPE);
}
}
}
else
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
}
}
goto done;
}
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>)
{
for (size_t i = 0; i < input_rows_count; ++i)
{
bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]);
@ -282,14 +321,38 @@ struct ConvertImpl
}
}
}
}
else
{
if constexpr (std::is_same_v<FromDataType, DataTypeUInt64> && std::is_same_v<ToDataType, DataTypeFloat32>)
{
/// Turns out that when ClickHouse is compiled with AVX1 or AVX2 instructions, Clang's autovectorizer produces
/// code for UInt64-to-Float23 conversion which is only ~50% as fast as scalar code. Interestingly, scalar code
/// is equally fast than code compiled for SSE4.2, so we might as well disable vectorization. This situation
/// may change with AVX512 which has a dediated instruction for that usecase (_mm512_cvtepi64_ps).
#if defined(__x86_64__)
# ifdef __clang__
# pragma clang loop vectorize(disable) interleave(disable)
# endif
#endif
for (size_t i = 0; i < input_rows_count; ++i)
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
}
else
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
for (size_t i = 0; i < input_rows_count; ++i)
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
}
}
}
}
done:
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
return ColumnNullable::create(std::move(col_to), std::move(col_null_map_to));
else

View File

@ -5,7 +5,7 @@
namespace DB
{
void registerFunctionsEmbeddedDictionaries(FunctionFactory & factory)
REGISTER_FUNCTION(EmbeddedDictionaries)
{
factory.registerFunction<FunctionRegionToCity>();
factory.registerFunction<FunctionRegionToArea>();

View File

@ -5,7 +5,7 @@
namespace DB
{
void registerFunctionsExternalDictionaries(FunctionFactory & factory)
REGISTER_FUNCTION(ExternalDictionaries)
{
factory.registerFunction<FunctionDictHas>();
factory.registerFunction<FunctionDictGetUInt8>();

View File

@ -6,7 +6,7 @@
namespace DB
{
void registerFunctionsHashing(FunctionFactory & factory)
REGISTER_FUNCTION(Hashing)
{
#if USE_SSL
factory.registerFunction<FunctionMD4>();

View File

@ -1443,7 +1443,7 @@ public:
}
};
void registerFunctionsJSON(FunctionFactory & factory)
REGISTER_FUNCTION(JSON)
{
factory.registerFunction<JSONOverloadResolver<NameJSONHas, JSONHasImpl>>();
factory.registerFunction<JSONOverloadResolver<NameIsValidJSON, IsValidJSONImpl>>();

View File

@ -221,7 +221,7 @@ struct NameDetectLanguage
using FunctionDetectLanguage = FunctionTextClassificationString<FunctionDetectLanguageImpl, NameDetectLanguage>;
void registerFunctionsDetectLanguage(FunctionFactory & factory)
REGISTER_FUNCTION(DetectLanguage)
{
factory.registerFunction<FunctionDetectLanguage>();
factory.registerFunction<FunctionDetectLanguageMixed>();

View File

@ -22,7 +22,7 @@
namespace DB
{
void registerFunctionsLogical(FunctionFactory & factory)
REGISTER_FUNCTION(Logical)
{
factory.registerFunction<FunctionAnd>();
factory.registerFunction<FunctionOr>();

View File

@ -112,7 +112,7 @@ struct NameDetectProgrammingLanguage
using FunctionDetectProgrammingLanguage = FunctionTextClassificationString<FunctionDetectProgrammingLanguageImpl, NameDetectProgrammingLanguage>;
void registerFunctionDetectProgrammingLanguage(FunctionFactory & factory)
REGISTER_FUNCTION(DetectProgrammingLanguage)
{
factory.registerFunction<FunctionDetectProgrammingLanguage>();
}

View File

@ -5,7 +5,7 @@
namespace DB
{
void registerFunctionsRound(FunctionFactory & factory)
REGISTER_FUNCTION(Round)
{
factory.registerFunction<FunctionRound>("round", FunctionFactory::CaseInsensitive);
factory.registerFunction<FunctionRoundBankers>("roundBankers", FunctionFactory::CaseSensitive);

View File

@ -27,7 +27,7 @@ DataTypePtr FunctionArrayStringConcat::getReturnTypeImpl(const DataTypes & argum
return std::make_shared<DataTypeString>();
}
void registerFunctionsStringArray(FunctionFactory & factory)
REGISTER_FUNCTION(StringArray)
{
factory.registerFunction<FunctionExtractAll>();
factory.registerFunction<FunctionAlphaTokens>();

View File

@ -743,7 +743,7 @@ using FunctionWordShingleMinHashArgUTF8
using FunctionWordShingleMinHashArgCaseInsensitiveUTF8
= FunctionsStringHash<MinHashImpl<true, false, true>, NameWordShingleMinHashArgCaseInsensitiveUTF8, false, true>;
void registerFunctionsStringHash(FunctionFactory & factory)
REGISTER_FUNCTION(StringHash)
{
factory.registerFunction<FunctionNgramSimHash>();
factory.registerFunction<FunctionNgramSimHashCaseInsensitive>();

View File

@ -530,7 +530,7 @@ using FunctionNgramSearchUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, U
using FunctionNgramSearchCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, false>, NameNgramSearchUTF8CaseInsensitive>;
void registerFunctionsStringSimilarity(FunctionFactory & factory)
REGISTER_FUNCTION(StringSimilarity)
{
factory.registerFunction<FunctionNgramDistance>();
factory.registerFunction<FunctionNgramDistanceCaseInsensitive>();

View File

@ -684,7 +684,7 @@ ColumnPtr FunctionTimeWindow<type>::executeImpl(const ColumnsWithTypeAndName & a
return TimeWindowImpl<type>::dispatchForColumns(arguments, name);
}
void registerFunctionsTimeWindow(FunctionFactory& factory)
REGISTER_FUNCTION(TimeWindow)
{
factory.registerFunction<FunctionTumble>();
factory.registerFunction<FunctionHop>();

View File

@ -81,7 +81,7 @@ struct NameDetectTonality
using FunctionDetectTonality = FunctionTextClassificationFloat<FunctionDetectTonalityImpl, NameDetectTonality>;
void registerFunctionDetectTonality(FunctionFactory & factory)
REGISTER_FUNCTION(DetectTonality)
{
factory.registerFunction<FunctionDetectTonality>();
}

View File

@ -61,7 +61,7 @@ public:
}
void registerFunctionsTransactionCounters(FunctionFactory & factory)
REGISTER_FUNCTION(TransactionCounters)
{
factory.registerFunction<FunctionTransactionID>();
factory.registerFunction<FunctionTransactionLatestSnapshot>();

View File

@ -6,7 +6,6 @@ add_library(clickhouse_functions_jsonpath ${clickhouse_functions_jsonpath_source
target_link_libraries(clickhouse_functions_jsonpath PRIVATE dbms)
target_link_libraries(clickhouse_functions_jsonpath PRIVATE clickhouse_parsers)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath)
if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options(clickhouse_functions_jsonpath PRIVATE "-g0")

View File

@ -6,19 +6,19 @@ namespace DB
{
using FunctionSubtractNanoseconds = FunctionDateOrDateTimeAddInterval<SubtractNanosecondsImpl>;
void registerFunctionSubtractNanoseconds(FunctionFactory & factory)
REGISTER_FUNCTION(SubtractNanoseconds)
{
factory.registerFunction<FunctionSubtractNanoseconds>();
}
using FunctionSubtractMicroseconds = FunctionDateOrDateTimeAddInterval<SubtractMicrosecondsImpl>;
void registerFunctionSubtractMicroseconds(FunctionFactory & factory)
REGISTER_FUNCTION(SubtractMicroseconds)
{
factory.registerFunction<FunctionSubtractMicroseconds>();
}
using FunctionSubtractMilliseconds = FunctionDateOrDateTimeAddInterval<SubtractMillisecondsImpl>;
void registerFunctionSubtractMilliseconds(FunctionFactory & factory)
REGISTER_FUNCTION(SubtractMilliseconds)
{
factory.registerFunction<FunctionSubtractMilliseconds>();
}

View File

@ -1,6 +1,6 @@
include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
add_headers_and_sources(clickhouse_functions_url .)
add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
add_library(clickhouse_functions_url OBJECT ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
target_link_libraries(clickhouse_functions_url PRIVATE dbms)
if (OMIT_HEAVY_DEBUG_SYMBOLS)
@ -10,6 +10,7 @@ endif()
# TODO: move Functions/Regexps.h to some lib and use here
if (TARGET ch_contrib::vectorscan)
target_link_libraries(clickhouse_functions_url PRIVATE ch_contrib::vectorscan)
list (APPEND PRIVATE_LIBS ch_contrib::vectorscan PARENT_SCOPE)
endif()
if (USE_GPERF)

View File

@ -104,7 +104,7 @@ public:
struct NameURLPathHierarchy { static constexpr auto name = "URLPathHierarchy"; };
using FunctionURLPathHierarchy = FunctionTokens<URLPathHierarchyImpl>;
void registerFunctionURLPathHierarchy(FunctionFactory & factory)
REGISTER_FUNCTION(URLPathHierarchy)
{
factory.registerFunction<FunctionURLPathHierarchy>();
}

View File

@ -106,7 +106,7 @@ public:
struct NameURLHierarchy { static constexpr auto name = "URLHierarchy"; };
using FunctionURLHierarchy = FunctionTokens<URLHierarchyImpl>;
void registerFunctionURLHierarchy(FunctionFactory & factory)
REGISTER_FUNCTION(URLHierarchy)
{
factory.registerFunction<FunctionURLHierarchy>();
}

View File

@ -34,7 +34,7 @@ struct ExtractBasename
struct NameBasename { static constexpr auto name = "basename"; };
using FunctionBasename = FunctionStringToString<ExtractSubstringImpl<ExtractBasename>, NameBasename>;
void registerFunctionBasename(FunctionFactory & factory)
REGISTER_FUNCTION(Basename)
{
factory.registerFunction<FunctionBasename>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameCutFragment { static constexpr auto name = "cutFragment"; };
using FunctionCutFragment = FunctionStringToString<CutSubstringImpl<ExtractFragment<false>>, NameCutFragment>;
void registerFunctionCutFragment(FunctionFactory & factory)
REGISTER_FUNCTION(CutFragment)
{
factory.registerFunction<FunctionCutFragment>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameCutQueryString { static constexpr auto name = "cutQueryString"; };
using FunctionCutQueryString = FunctionStringToString<CutSubstringImpl<ExtractQueryString<false>>, NameCutQueryString>;
void registerFunctionCutQueryString(FunctionFactory & factory)
REGISTER_FUNCTION(CutQueryString)
{
factory.registerFunction<FunctionCutQueryString>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameCutQueryStringAndFragment { static constexpr auto name = "cutQueryStringAndFragment"; };
using FunctionCutQueryStringAndFragment = FunctionStringToString<CutSubstringImpl<ExtractQueryStringAndFragment<false>>, NameCutQueryStringAndFragment>;
void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory)
REGISTER_FUNCTION(CutQueryStringAndFragment)
{
factory.registerFunction<FunctionCutQueryStringAndFragment>();
}

View File

@ -35,7 +35,7 @@ using FunctionCutToFirstSignificantSubdomain = FunctionStringToString<ExtractSub
struct NameCutToFirstSignificantSubdomainWithWWW { static constexpr auto name = "cutToFirstSignificantSubdomainWithWWW"; };
using FunctionCutToFirstSignificantSubdomainWithWWW = FunctionStringToString<ExtractSubstringImpl<CutToFirstSignificantSubdomain<false>>, NameCutToFirstSignificantSubdomainWithWWW>;
void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory & factory)
REGISTER_FUNCTION(CutToFirstSignificantSubdomain)
{
factory.registerFunction<FunctionCutToFirstSignificantSubdomain>();
factory.registerFunction<FunctionCutToFirstSignificantSubdomainWithWWW>();

View File

@ -34,7 +34,7 @@ using FunctionCutToFirstSignificantSubdomainCustom = FunctionCutToFirstSignifica
struct NameCutToFirstSignificantSubdomainCustomWithWWW { static constexpr auto name = "cutToFirstSignificantSubdomainCustomWithWWW"; };
using FunctionCutToFirstSignificantSubdomainCustomWithWWW = FunctionCutToFirstSignificantSubdomainCustomImpl<CutToFirstSignificantSubdomainCustom<false>, NameCutToFirstSignificantSubdomainCustomWithWWW>;
void registerFunctionCutToFirstSignificantSubdomainCustom(FunctionFactory & factory)
REGISTER_FUNCTION(CutToFirstSignificantSubdomainCustom)
{
factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustom>();
factory.registerFunction<FunctionCutToFirstSignificantSubdomainCustomWithWWW>();

View File

@ -77,7 +77,7 @@ struct CutURLParameterImpl
struct NameCutURLParameter { static constexpr auto name = "cutURLParameter"; };
using FunctionCutURLParameter = FunctionsStringSearchToString<CutURLParameterImpl, NameCutURLParameter>;
void registerFunctionCutURLParameter(FunctionFactory & factory)
REGISTER_FUNCTION(CutURLParameter)
{
factory.registerFunction<FunctionCutURLParameter>();
}

View File

@ -54,7 +54,7 @@ struct ExtractWWW
struct NameCutWWW { static constexpr auto name = "cutWWW"; };
using FunctionCutWWW = FunctionStringToString<CutSubstringImpl<ExtractWWW>, NameCutWWW>;
void registerFunctionCutWWW(FunctionFactory & factory)
REGISTER_FUNCTION(CutWWW)
{
factory.registerFunction<FunctionCutWWW>();
}

View File

@ -172,7 +172,7 @@ using FunctionEncodeURLComponent = FunctionStringToString<CodeURLComponentImpl<e
using FunctionDecodeURLFormComponent = FunctionStringToString<CodeURLComponentImpl<decode, true>, NameDecodeURLFormComponent>;
using FunctionEncodeURLFormComponent = FunctionStringToString<CodeURLComponentImpl<encode, true>, NameEncodeURLFormComponent>;
void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory)
REGISTER_FUNCTION(EncodeAndDecodeURLComponent)
{
factory.registerFunction<FunctionDecodeURLComponent>();
factory.registerFunction<FunctionEncodeURLComponent>();

View File

@ -10,7 +10,7 @@ struct NameDomain { static constexpr auto name = "domain"; };
using FunctionDomain = FunctionStringToString<ExtractSubstringImpl<ExtractDomain<false>>, NameDomain>;
void registerFunctionDomain(FunctionFactory & factory)
REGISTER_FUNCTION(Domain)
{
factory.registerFunction<FunctionDomain>();
}

View File

@ -9,7 +9,7 @@ struct NameDomainWithoutWWW { static constexpr auto name = "domainWithoutWWW"; }
using FunctionDomainWithoutWWW = FunctionStringToString<ExtractSubstringImpl<ExtractDomain<true>>, NameDomainWithoutWWW>;
void registerFunctionDomainWithoutWWW(FunctionFactory & factory)
REGISTER_FUNCTION(DomainWithoutWWW)
{
factory.registerFunction<FunctionDomainWithoutWWW>();
}

View File

@ -92,7 +92,7 @@ struct ExtractURLParameterImpl
struct NameExtractURLParameter { static constexpr auto name = "extractURLParameter"; };
using FunctionExtractURLParameter = FunctionsStringSearchToString<ExtractURLParameterImpl, NameExtractURLParameter>;
void registerFunctionExtractURLParameter(FunctionFactory & factory)
REGISTER_FUNCTION(ExtractURLParameter)
{
factory.registerFunction<FunctionExtractURLParameter>();
}

View File

@ -89,7 +89,7 @@ public:
struct NameExtractURLParameterNames { static constexpr auto name = "extractURLParameterNames"; };
using FunctionExtractURLParameterNames = FunctionTokens<ExtractURLParameterNamesImpl>;
void registerFunctionExtractURLParameterNames(FunctionFactory & factory)
REGISTER_FUNCTION(ExtractURLParameterNames)
{
factory.registerFunction<FunctionExtractURLParameterNames>();
}

View File

@ -97,7 +97,7 @@ public:
struct NameExtractURLParameters { static constexpr auto name = "extractURLParameters"; };
using FunctionExtractURLParameters = FunctionTokens<ExtractURLParametersImpl>;
void registerFunctionExtractURLParameters(FunctionFactory & factory)
REGISTER_FUNCTION(ExtractURLParameters)
{
factory.registerFunction<FunctionExtractURLParameters>();
}

View File

@ -10,7 +10,7 @@ struct NameFirstSignificantSubdomain { static constexpr auto name = "firstSignif
using FunctionFirstSignificantSubdomain = FunctionStringToString<ExtractSubstringImpl<ExtractFirstSignificantSubdomain<true>>, NameFirstSignificantSubdomain>;
void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory)
REGISTER_FUNCTION(FirstSignificantSubdomain)
{
factory.registerFunction<FunctionFirstSignificantSubdomain>();
}

View File

@ -10,7 +10,7 @@ struct NameFirstSignificantSubdomainCustom { static constexpr auto name = "first
using FunctionFirstSignificantSubdomainCustom = FunctionCutToFirstSignificantSubdomainCustomImpl<ExtractFirstSignificantSubdomain<true>, NameFirstSignificantSubdomainCustom>;
void registerFunctionFirstSignificantSubdomainCustom(FunctionFactory & factory)
REGISTER_FUNCTION(FirstSignificantSubdomainCustom)
{
factory.registerFunction<FunctionFirstSignificantSubdomainCustom>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameFragment { static constexpr auto name = "fragment"; };
using FunctionFragment = FunctionStringToString<ExtractSubstringImpl<ExtractFragment<true>>, NameFragment>;
void registerFunctionFragment(FunctionFactory & factory)
REGISTER_FUNCTION(Fragment)
{
factory.registerFunction<FunctionFragment>();
}

View File

@ -132,7 +132,7 @@ struct ExtractNetloc
struct NameNetloc { static constexpr auto name = "netloc"; };
using FunctionNetloc = FunctionStringToString<ExtractSubstringImpl<ExtractNetloc>, NameNetloc>;
void registerFunctionNetloc(FunctionFactory & factory)
REGISTER_FUNCTION(Netloc)
{
factory.registerFunction<FunctionNetloc>();
}

View File

@ -11,7 +11,7 @@ namespace DB
struct NamePath { static constexpr auto name = "path"; };
using FunctionPath = FunctionStringToString<ExtractSubstringImpl<ExtractPath<false>>, NamePath>;
void registerFunctionPath(FunctionFactory & factory)
REGISTER_FUNCTION(Path)
{
factory.registerFunction<FunctionPath>();
}

View File

@ -10,7 +10,7 @@ namespace DB
struct NamePathFull { static constexpr auto name = "pathFull"; };
using FunctionPathFull = FunctionStringToString<ExtractSubstringImpl<ExtractPath<true>>, NamePathFull>;
void registerFunctionPathFull(FunctionFactory & factory)
REGISTER_FUNCTION(PathFull)
{
factory.registerFunction<FunctionPathFull>();
}

View File

@ -121,7 +121,7 @@ private:
}
};
void registerFunctionPort(FunctionFactory & factory)
REGISTER_FUNCTION(Port)
{
factory.registerFunction<FunctionPort>();
}

View File

@ -9,7 +9,7 @@ namespace DB
struct NameProtocol { static constexpr auto name = "protocol"; };
using FunctionProtocol = FunctionStringToString<ExtractSubstringImpl<ExtractProtocol>, NameProtocol>;
void registerFunctionProtocol(FunctionFactory & factory)
REGISTER_FUNCTION(Protocol)
{
factory.registerFunction<FunctionProtocol>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameQueryString { static constexpr auto name = "queryString"; };
using FunctionQueryString = FunctionStringToString<ExtractSubstringImpl<ExtractQueryString<true>>, NameQueryString>;
void registerFunctionQueryString(FunctionFactory & factory)
REGISTER_FUNCTION(QueryString)
{
factory.registerFunction<FunctionQueryString>();
}

View File

@ -8,7 +8,7 @@ namespace DB
struct NameQueryStringAndFragment { static constexpr auto name = "queryStringAndFragment"; };
using FunctionQueryStringAndFragment = FunctionStringToString<ExtractSubstringImpl<ExtractQueryStringAndFragment<true>>, NameQueryStringAndFragment>;
void registerFunctionQueryStringAndFragment(FunctionFactory & factory)
REGISTER_FUNCTION(QueryStringAndFragment)
{
factory.registerFunction<FunctionQueryStringAndFragment>();
}

View File

@ -1,64 +0,0 @@
namespace DB
{
class FunctionFactory;
void registerFunctionProtocol(FunctionFactory & factory);
void registerFunctionDomain(FunctionFactory & factory);
void registerFunctionDomainWithoutWWW(FunctionFactory & factory);
void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory);
void registerFunctionFirstSignificantSubdomainCustom(FunctionFactory & factory);
void registerFunctionTopLevelDomain(FunctionFactory & factory);
void registerFunctionPort(FunctionFactory & factory);
void registerFunctionPath(FunctionFactory & factory);
void registerFunctionPathFull(FunctionFactory & factory);
void registerFunctionQueryString(FunctionFactory & factory);
void registerFunctionFragment(FunctionFactory & factory);
void registerFunctionQueryStringAndFragment(FunctionFactory & factory);
void registerFunctionExtractURLParameter(FunctionFactory & factory);
void registerFunctionExtractURLParameters(FunctionFactory & factory);
void registerFunctionExtractURLParameterNames(FunctionFactory & factory);
void registerFunctionURLHierarchy(FunctionFactory & factory);
void registerFunctionURLPathHierarchy(FunctionFactory & factory);
void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory & factory);
void registerFunctionCutToFirstSignificantSubdomainCustom(FunctionFactory & factory);
void registerFunctionCutWWW(FunctionFactory & factory);
void registerFunctionCutQueryString(FunctionFactory & factory);
void registerFunctionCutFragment(FunctionFactory & factory);
void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory);
void registerFunctionCutURLParameter(FunctionFactory & factory);
void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory);
void registerFunctionNetloc(FunctionFactory & factory);
void registerFunctionsURL(FunctionFactory & factory)
{
registerFunctionProtocol(factory);
registerFunctionDomain(factory);
registerFunctionDomainWithoutWWW(factory);
registerFunctionFirstSignificantSubdomain(factory);
registerFunctionFirstSignificantSubdomainCustom(factory);
registerFunctionTopLevelDomain(factory);
registerFunctionPort(factory);
registerFunctionPath(factory);
registerFunctionPathFull(factory);
registerFunctionQueryString(factory);
registerFunctionFragment(factory);
registerFunctionQueryStringAndFragment(factory);
registerFunctionExtractURLParameter(factory);
registerFunctionExtractURLParameters(factory);
registerFunctionExtractURLParameterNames(factory);
registerFunctionURLHierarchy(factory);
registerFunctionURLPathHierarchy(factory);
registerFunctionCutToFirstSignificantSubdomain(factory);
registerFunctionCutToFirstSignificantSubdomainCustom(factory);
registerFunctionCutWWW(factory);
registerFunctionCutQueryString(factory);
registerFunctionCutFragment(factory);
registerFunctionCutQueryStringAndFragment(factory);
registerFunctionCutURLParameter(factory);
registerFunctionEncodeAndDecodeURLComponent(factory);
registerFunctionNetloc(factory);
}
}

View File

@ -43,7 +43,7 @@ struct ExtractTopLevelDomain
struct NameTopLevelDomain { static constexpr auto name = "topLevelDomain"; };
using FunctionTopLevelDomain = FunctionStringToString<ExtractSubstringImpl<ExtractTopLevelDomain>, NameTopLevelDomain>;
void registerFunctionTopLevelDomain(FunctionFactory & factory)
REGISTER_FUNCTION(TopLevelDomain)
{
factory.registerFunction<FunctionTopLevelDomain>();
}

View File

@ -50,7 +50,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity<NameAbs>
}
};
void registerFunctionAbs(FunctionFactory & factory)
REGISTER_FUNCTION(Abs)
{
factory.registerFunction<FunctionAbs>(FunctionFactory::CaseInsensitive);
}

View File

@ -12,7 +12,7 @@ using FunctionAcos = FunctionMathUnary<UnaryFunctionVectorized<AcosName, acos>>;
}
void registerFunctionAcos(FunctionFactory & factory)
REGISTER_FUNCTION(Acos)
{
factory.registerFunction<FunctionAcos>(FunctionFactory::CaseInsensitive);
}

View File

@ -13,7 +13,7 @@ namespace
}
void registerFunctionAcosh(FunctionFactory & factory)
REGISTER_FUNCTION(Acosh)
{
factory.registerFunction<FunctionAcosh>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddDays = FunctionDateOrDateTimeAddInterval<AddDaysImpl>;
void registerFunctionAddDays(FunctionFactory & factory)
REGISTER_FUNCTION(AddDays)
{
factory.registerFunction<FunctionAddDays>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddHours = FunctionDateOrDateTimeAddInterval<AddHoursImpl>;
void registerFunctionAddHours(FunctionFactory & factory)
REGISTER_FUNCTION(AddHours)
{
factory.registerFunction<FunctionAddHours>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddMinutes = FunctionDateOrDateTimeAddInterval<AddMinutesImpl>;
void registerFunctionAddMinutes(FunctionFactory & factory)
REGISTER_FUNCTION(AddMinutes)
{
factory.registerFunction<FunctionAddMinutes>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddMonths = FunctionDateOrDateTimeAddInterval<AddMonthsImpl>;
void registerFunctionAddMonths(FunctionFactory & factory)
REGISTER_FUNCTION(AddMonths)
{
factory.registerFunction<FunctionAddMonths>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddQuarters = FunctionDateOrDateTimeAddInterval<AddQuartersImpl>;
void registerFunctionAddQuarters(FunctionFactory & factory)
REGISTER_FUNCTION(AddQuarters)
{
factory.registerFunction<FunctionAddQuarters>();
}

View File

@ -7,7 +7,7 @@ namespace DB
using FunctionAddSeconds = FunctionDateOrDateTimeAddInterval<AddSecondsImpl>;
void registerFunctionAddSeconds(FunctionFactory & factory)
REGISTER_FUNCTION(AddSeconds)
{
factory.registerFunction<FunctionAddSeconds>();
}

Some files were not shown because too many files have changed in this diff Show More