mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 15:42:02 +00:00
Merge remote-tracking branch 'origin/master' into tmp
This commit is contained in:
Alexander Kuzmenkov
commit
e729dc5552
10
CHANGELOG.md
10
CHANGELOG.md
@ -22,14 +22,14 @@
|
||||
* Add setting `allow_non_metadata_alters` which restricts to execute `ALTER` queries which modify data on disk. Disabled be default. Closes [#11547](https://github.com/ClickHouse/ClickHouse/issues/11547). [#12635](https://github.com/ClickHouse/ClickHouse/pull/12635) ([alesapin](https://github.com/alesapin)).
|
||||
* A function `formatRow` is added to support turning arbitrary expressions into a string via given format. It's useful for manipulating SQL outputs and is quite versatile combined with the `columns` function. [#12574](https://github.com/ClickHouse/ClickHouse/pull/12574) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Add `FROM_UNIXTIME` function for compatibility with MySQL, related to [12149](https://github.com/ClickHouse/ClickHouse/issues/12149). [#12484](https://github.com/ClickHouse/ClickHouse/pull/12484) ([flynn](https://github.com/ucasFL)).
|
||||
* Allow Nullable types as keys in MergeTree tables if `allow_nullable_key` table setting is enabled. https://github.com/ClickHouse/ClickHouse/issues/5319. [#12433](https://github.com/ClickHouse/ClickHouse/pull/12433) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Allow Nullable types as keys in MergeTree tables if `allow_nullable_key` table setting is enabled. Closes [#5319](https://github.com/ClickHouse/ClickHouse/issues/5319). [#12433](https://github.com/ClickHouse/ClickHouse/pull/12433) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Integration with [COS](https://intl.cloud.tencent.com/product/cos). [#12386](https://github.com/ClickHouse/ClickHouse/pull/12386) ([fastio](https://github.com/fastio)).
|
||||
* Add mapAdd and mapSubtract functions for adding/subtracting key-mapped values. [#11735](https://github.com/ClickHouse/ClickHouse/pull/11735) ([Ildus Kurbangaliev](https://github.com/ildus)).
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix premature `ON CLUSTER` timeouts for queries that must be executed on a single replica. Fixes [#6704](https://github.com/ClickHouse/ClickHouse/issues/6704), [#7228](https://github.com/ClickHouse/ClickHouse/issues/7228), [#13361](https://github.com/ClickHouse/ClickHouse/issues/13361), [#11884](https://github.com/ClickHouse/ClickHouse/issues/11884). [#13450](https://github.com/ClickHouse/ClickHouse/pull/13450) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix crash in mark inclusion search introduced in https://github.com/ClickHouse/ClickHouse/pull/12277. [#14225](https://github.com/ClickHouse/ClickHouse/pull/14225) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix crash in mark inclusion search introduced in [#12277](https://github.com/ClickHouse/ClickHouse/pull/12277). [#14225](https://github.com/ClickHouse/ClickHouse/pull/14225) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix race condition in external dictionaries with cache layout which can lead server crash. [#12566](https://github.com/ClickHouse/ClickHouse/pull/12566) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix visible data clobbering by progress bar in client in interactive mode. This fixes [#12562](https://github.com/ClickHouse/ClickHouse/issues/12562) and [#13369](https://github.com/ClickHouse/ClickHouse/issues/13369) and [#13584](https://github.com/ClickHouse/ClickHouse/issues/13584) and fixes [#12964](https://github.com/ClickHouse/ClickHouse/issues/12964). [#13691](https://github.com/ClickHouse/ClickHouse/pull/13691) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fixed incorrect sorting order for `LowCardinality` columns when ORDER BY multiple columns is used. This fixes [#13958](https://github.com/ClickHouse/ClickHouse/issues/13958). [#14223](https://github.com/ClickHouse/ClickHouse/pull/14223) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
@ -71,7 +71,7 @@
|
||||
* Fix function if with nullable constexpr as cond that is not literal NULL. Fixes [#12463](https://github.com/ClickHouse/ClickHouse/issues/12463). [#13226](https://github.com/ClickHouse/ClickHouse/pull/13226) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix assert in `arrayElement` function in case of array elements are Nullable and array subscript is also Nullable. This fixes [#12172](https://github.com/ClickHouse/ClickHouse/issues/12172). [#13224](https://github.com/ClickHouse/ClickHouse/pull/13224) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix DateTime64 conversion functions with constant argument. [#13205](https://github.com/ClickHouse/ClickHouse/pull/13205) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes https://github.com/ClickHouse/ClickHouse/issues/5779, https://github.com/ClickHouse/ClickHouse/issues/12527. [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix access to `redis` dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix wrong index analysis with functions. It could lead to some data parts being skipped when reading from `MergeTree` tables. Fixes [#13060](https://github.com/ClickHouse/ClickHouse/issues/13060). Fixes [#12406](https://github.com/ClickHouse/ClickHouse/issues/12406). [#13081](https://github.com/ClickHouse/ClickHouse/pull/13081) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix error `Cannot convert column because it is constant but values of constants are different in source and result` for remote queries which use deterministic functions in scope of query, but not deterministic between queries, like `now()`, `now64()`, `randConstant()`. Fixes [#11327](https://github.com/ClickHouse/ClickHouse/issues/11327). [#13075](https://github.com/ClickHouse/ClickHouse/pull/13075) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
@ -89,7 +89,7 @@
|
||||
* Fixed [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572) fix bloom filter index with const expression. [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)).
|
||||
* Fix SIGSEGV in StorageKafka when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Add support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes https://github.com/ClickHouse/ClickHouse/issues/12507. [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Exception `There is no supertype...` can be thrown during `ALTER ... UPDATE` in unexpected cases (e.g. when subtracting from UInt64 column). This fixes [#7306](https://github.com/ClickHouse/ClickHouse/issues/7306). This fixes [#4165](https://github.com/ClickHouse/ClickHouse/issues/4165). [#12633](https://github.com/ClickHouse/ClickHouse/pull/12633) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix possible `Pipeline stuck` error for queries with external sorting. Fixes [#12617](https://github.com/ClickHouse/ClickHouse/issues/12617). [#12618](https://github.com/ClickHouse/ClickHouse/pull/12618) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix error `Output of TreeExecutor is not sorted` for `OPTIMIZE DEDUPLICATE`. Fixes [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572). [#12613](https://github.com/ClickHouse/ClickHouse/pull/12613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
@ -123,7 +123,7 @@
|
||||
* Fix assert in `parseDateTimeBestEffort`. This fixes [#12649](https://github.com/ClickHouse/ClickHouse/issues/12649). [#13227](https://github.com/ClickHouse/ClickHouse/pull/13227) ([alexey-milovidov](https://github.com/alexey-milovidov)).
|
||||
* Minor optimization in Processors/PipelineExecutor: breaking out of a loop because it makes sense to do so. [#13058](https://github.com/ClickHouse/ClickHouse/pull/13058) ([Mark Papadakis](https://github.com/markpapadakis)).
|
||||
* Support TRUNCATE table without TABLE keyword. [#12653](https://github.com/ClickHouse/ClickHouse/pull/12653) ([Winter Zhang](https://github.com/zhang2014)).
|
||||
* Fix explain query format overwrite by default, issue https://github.com/ClickHouse/ClickHouse/issues/12432. [#12541](https://github.com/ClickHouse/ClickHouse/pull/12541) ([BohuTANG](https://github.com/BohuTANG)).
|
||||
* Fix explain query format overwrite by default. This fixes [#12541](https://github.com/ClickHouse/ClickHouse/issues/12432). [#12541](https://github.com/ClickHouse/ClickHouse/pull/12541) ([BohuTANG](https://github.com/BohuTANG)).
|
||||
* Allow to set JOIN kind and type in more standad way: `LEFT SEMI JOIN` instead of `SEMI LEFT JOIN`. For now both are correct. [#12520](https://github.com/ClickHouse/ClickHouse/pull/12520) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Changes default value for `multiple_joins_rewriter_version` to 2. It enables new multiple joins rewriter that knows about column names. [#12469](https://github.com/ClickHouse/ClickHouse/pull/12469) ([Artem Zuikov](https://github.com/4ertus2)).
|
||||
* Add several metrics for requests to S3 storages. [#12464](https://github.com/ClickHouse/ClickHouse/pull/12464) ([ianton-ru](https://github.com/ianton-ru)).
|
||||
|
||||
@ -18,3 +18,4 @@ ClickHouse is an open-source column-oriented database management system that all
|
||||
## Upcoming Events
|
||||
|
||||
* [ClickHouse Data Integration Virtual Meetup](https://www.eventbrite.com/e/clickhouse-september-virtual-meetup-data-integration-tickets-117421895049) on September 10, 2020.
|
||||
* [ClickHouse talk at Ya.Subbotnik (in Russian)](https://ya.cc/t/cIBI-3yECj5JF) on September 12, 2020.
|
||||
|
||||
@ -1,9 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <mysqlxx/Types.h>
|
||||
|
||||
|
||||
namespace mysqlxx
|
||||
{
|
||||
|
||||
@ -22,6 +20,11 @@ class ResultBase
|
||||
public:
|
||||
ResultBase(MYSQL_RES * res_, Connection * conn_, const Query * query_);
|
||||
|
||||
ResultBase(const ResultBase &) = delete;
|
||||
ResultBase & operator=(const ResultBase &) = delete;
|
||||
ResultBase(ResultBase &&) = default;
|
||||
ResultBase & operator=(ResultBase &&) = default;
|
||||
|
||||
Connection * getConnection() { return conn; }
|
||||
MYSQL_FIELDS getFields() { return fields; }
|
||||
unsigned getNumFields() { return num_fields; }
|
||||
|
||||
@ -254,7 +254,23 @@ template <> inline std::string Value::get<std::string >() cons
|
||||
template <> inline LocalDate Value::get<LocalDate >() const { return getDate(); }
|
||||
template <> inline LocalDateTime Value::get<LocalDateTime >() const { return getDateTime(); }
|
||||
|
||||
template <typename T> inline T Value::get() const { return T(*this); }
|
||||
|
||||
namespace details
|
||||
{
|
||||
// To avoid stack overflow when converting to type with no appropriate c-tor,
|
||||
// resulting in endless recursive calls from `Value::get<T>()` to `Value::operator T()` to `Value::get<T>()` to ...
|
||||
template <typename T, typename std::enable_if_t<std::is_constructible_v<T, Value>>>
|
||||
inline T contructFromValue(const Value & val)
|
||||
{
|
||||
return T(val);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline T Value::get() const
|
||||
{
|
||||
return details::contructFromValue<T>(*this);
|
||||
}
|
||||
|
||||
|
||||
inline std::ostream & operator<< (std::ostream & ostr, const Value & x)
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
SET(VERSION_REVISION 54439)
|
||||
SET(VERSION_REVISION 54440)
|
||||
SET(VERSION_MAJOR 20)
|
||||
SET(VERSION_MINOR 9)
|
||||
SET(VERSION_MINOR 10)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH 0586f0d555f7481b394afc55bbb29738cd573a1c)
|
||||
SET(VERSION_DESCRIBE v20.9.1.1-prestable)
|
||||
SET(VERSION_STRING 20.9.1.1)
|
||||
SET(VERSION_GITHASH 11a247d2f42010c1a17bf678c3e00a4bc89b23f8)
|
||||
SET(VERSION_DESCRIBE v20.10.1.1-prestable)
|
||||
SET(VERSION_STRING 20.10.1.1)
|
||||
# end of autochange
|
||||
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
||||
clickhouse (20.9.1.1) unstable; urgency=low
|
||||
clickhouse (20.10.1.1) unstable; urgency=low
|
||||
|
||||
* Modified source code
|
||||
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Mon, 31 Aug 2020 23:07:38 +0300
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Tue, 08 Sep 2020 17:04:39 +0300
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.9.1.*
|
||||
ARG version=20.10.1.*
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -18,7 +18,7 @@ ccache --zero-stats ||:
|
||||
ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
|
||||
rm -f CMakeCache.txt
|
||||
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS ..
|
||||
ninja $NINJA_FLAGS clickhouse-bundle
|
||||
ninja -j $(($(nproc) / 2)) $NINJA_FLAGS clickhouse-bundle
|
||||
mv ./programs/clickhouse* /output
|
||||
mv ./src/unit_tests_dbms /output
|
||||
find . -name '*.so' -print -exec mv '{}' /output \;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.9.1.*
|
||||
ARG version=20.10.1.*
|
||||
ARG gosu_ver=1.10
|
||||
|
||||
RUN apt-get update \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=20.9.1.*
|
||||
ARG version=20.10.1.*
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y apt-transport-https dirmngr && \
|
||||
|
||||
@ -5,4 +5,4 @@ services:
|
||||
restart: always
|
||||
ports:
|
||||
- 6380:6379
|
||||
command: redis-server --requirepass "clickhouse"
|
||||
command: redis-server --requirepass "clickhouse" --databases 32
|
||||
|
||||
@ -82,8 +82,8 @@ res: /lib/x86_64-linux-gnu/libc-2.27.so
|
||||
|
||||
- [Introspection Functions](../../sql-reference/functions/introspection.md) — Which introspection functions are available and how to use them.
|
||||
- [system.trace_log](../system-tables/trace_log.md) — Contains stack traces collected by the sampling query profiler.
|
||||
- [arrayMap](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-map) — Description and usage example of the `arrayMap` function.
|
||||
- [arrayFilter](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-filter) — Description and usage example of the `arrayFilter` function.
|
||||
- [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — Description and usage example of the `arrayMap` function.
|
||||
- [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — Description and usage example of the `arrayFilter` function.
|
||||
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system-tables/stack_trace) <!--hide-->
|
||||
|
||||
@ -7,7 +7,7 @@ toc_title: Tuple(T1, T2, ...)
|
||||
|
||||
A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types).
|
||||
|
||||
Tuples are used for temporary column grouping. Columns can be grouped when an IN expression is used in a query, and for specifying certain formal parameters of lambda functions. For more information, see the sections [IN operators](../../sql-reference/operators/in.md) and [Higher order functions](../../sql-reference/functions/higher-order-functions.md).
|
||||
Tuples are used for temporary column grouping. Columns can be grouped when an IN expression is used in a query, and for specifying certain formal parameters of lambda functions. For more information, see the sections [IN operators](../../sql-reference/operators/in.md) and [Higher order functions](../../sql-reference/functions/index.md#higher-order-functions).
|
||||
|
||||
Tuples can be the result of a query. In this case, for text formats other than JSON, values are comma-separated in brackets. In JSON formats, tuples are output as arrays (in square brackets).
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
---
|
||||
toc_priority: 35
|
||||
toc_priority: 34
|
||||
toc_title: Arithmetic
|
||||
---
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
---
|
||||
toc_priority: 46
|
||||
toc_priority: 35
|
||||
toc_title: Arrays
|
||||
---
|
||||
|
||||
# Functions for Working with Arrays {#functions-for-working-with-arrays}
|
||||
# Array Functions {#functions-for-working-with-arrays}
|
||||
|
||||
## empty {#function-empty}
|
||||
|
||||
@ -241,6 +241,12 @@ SELECT indexOf([1, 3, NULL, NULL], NULL)
|
||||
|
||||
Elements set to `NULL` are handled as normal values.
|
||||
|
||||
## arrayCount(\[func,\] arr1, …) {#array-count}
|
||||
|
||||
Returns the number of elements in the arr array for which func returns something other than 0. If ‘func’ is not specified, it returns the number of non-zero elements in the array.
|
||||
|
||||
Note that the `arrayCount` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
## countEqual(arr, x) {#countequalarr-x}
|
||||
|
||||
Returns the number of elements in the array equal to x. Equivalent to arrayCount (elem -\> elem = x, arr).
|
||||
@ -568,7 +574,7 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
|
||||
- `NaN` values are right before `NULL`.
|
||||
- `Inf` values are right before `NaN`.
|
||||
|
||||
Note that `arraySort` is a [higher-order function](../../sql-reference/functions/higher-order-functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array.
|
||||
Note that `arraySort` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array.
|
||||
|
||||
Let’s consider the following example:
|
||||
|
||||
@ -668,7 +674,7 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
|
||||
- `NaN` values are right before `NULL`.
|
||||
- `-Inf` values are right before `NaN`.
|
||||
|
||||
Note that the `arrayReverseSort` is a [higher-order function](../../sql-reference/functions/higher-order-functions.md). You can pass a lambda function to it as the first argument. Example is shown below.
|
||||
Note that the `arrayReverseSort` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. Example is shown below.
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
|
||||
@ -1120,7 +1126,205 @@ Result:
|
||||
``` text
|
||||
┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐
|
||||
│ 0.75 │
|
||||
└────────────────────────────────────────---──┘
|
||||
└───────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## arrayMap(func, arr1, …) {#array-map}
|
||||
|
||||
Returns an array obtained from the original application of the `func` function to each element in the `arr` array.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────┐
|
||||
│ [3,4,5] │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
The following example shows how to create a tuple of elements from different arrays:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────────┐
|
||||
│ [(1,4),(2,5),(3,6)] │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
Note that the `arrayMap` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayFilter(func, arr1, …) {#array-filter}
|
||||
|
||||
Returns an array containing only the elements in `arr1` for which `func` returns something other than 0.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ ['abc World'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
arrayFilter(
|
||||
(i, x) -> x LIKE '%World%',
|
||||
arrayEnumerate(arr),
|
||||
['Hello', 'abc World'] AS arr)
|
||||
AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─┐
|
||||
│ [2] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
Note that the `arrayFilter` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayFill(func, arr1, …) {#array-fill}
|
||||
|
||||
Scan through `arr1` from the first element to the last element and replace `arr1[i]` by `arr1[i - 1]` if `func` returns 0. The first element of `arr1` will not be replaced.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────────────────────────┐
|
||||
│ [1,1,3,11,12,12,12,5,6,14,14,14] │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Note that the `arrayFill` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayReverseFill(func, arr1, …) {#array-reverse-fill}
|
||||
|
||||
Scan through `arr1` from the last element to the first element and replace `arr1[i]` by `arr1[i + 1]` if `func` returns 0. The last element of `arr1` will not be replaced.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res────────────────────────────────┐
|
||||
│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │
|
||||
└────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Note that the `arrayReverseFilter` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arraySplit(func, arr1, …) {#array-split}
|
||||
|
||||
Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the left hand side of the element. The array will not be split before the first element.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────┐
|
||||
│ [[1,2,3],[4,5]] │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
Note that the `arraySplit` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayReverseSplit(func, arr1, …) {#array-reverse-split}
|
||||
|
||||
Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the right hand side of the element. The array will not be split after the last element.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────────┐
|
||||
│ [[1],[2,3,4],[5]] │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
Note that the `arrayReverseSplit` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1}
|
||||
|
||||
Returns 1 if there is at least one element in `arr` for which `func` returns something other than 0. Otherwise, it returns 0.
|
||||
|
||||
Note that the `arrayExists` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
## arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1}
|
||||
|
||||
Returns 1 if `func` returns something other than 0 for all the elements in `arr`. Otherwise, it returns 0.
|
||||
|
||||
Note that the `arrayAll` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
## arrayFirst(func, arr1, …) {#array-first}
|
||||
|
||||
Returns the first element in the `arr1` array for which `func` returns something other than 0.
|
||||
|
||||
Note that the `arrayFirst` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arrayFirstIndex(func, arr1, …) {#array-first-index}
|
||||
|
||||
Returns the index of the first element in the `arr1` array for which `func` returns something other than 0.
|
||||
|
||||
Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted.
|
||||
|
||||
## arraySum(\[func,\] arr1, …) {#array-sum}
|
||||
|
||||
Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements.
|
||||
|
||||
Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
## arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1}
|
||||
|
||||
Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSum([1, 1, 1, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────┐
|
||||
│ [1, 2, 3, 4] │
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
Note that the `arrayCumSum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
## arrayCumSumNonNegative(arr) {#arraycumsumnonnegativearr}
|
||||
|
||||
Same as `arrayCumSum`, returns an array of partial sums of elements in the source array (a running sum). Different `arrayCumSum`, when then returned value contains a value less than zero, the value is replace with zero and the subsequent calculation is performed with zero parameters. For example:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────┐
|
||||
│ [1,2,0,1] │
|
||||
└───────────┘
|
||||
```
|
||||
Note that the `arraySumNonNegative` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) <!--hide-->
|
||||
|
||||
@ -1,262 +0,0 @@
|
||||
---
|
||||
toc_priority: 57
|
||||
toc_title: Higher-Order
|
||||
---
|
||||
|
||||
# Higher-order Functions {#higher-order-functions}
|
||||
|
||||
## `->` operator, lambda(params, expr) function {#operator-lambdaparams-expr-function}
|
||||
|
||||
Allows describing a lambda function for passing to a higher-order function. The left side of the arrow has a formal parameter, which is any ID, or multiple formal parameters – any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns.
|
||||
|
||||
Examples: `x -> 2 * x, str -> str != Referer.`
|
||||
|
||||
Higher-order functions can only accept lambda functions as their functional argument.
|
||||
|
||||
A lambda function that accepts multiple arguments can be passed to a higher-order function. In this case, the higher-order function is passed several arrays of identical length that these arguments will correspond to.
|
||||
|
||||
For some functions, such as [arrayCount](#higher_order_functions-array-count) or [arraySum](#higher_order_functions-array-count), the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed.
|
||||
|
||||
A lambda function can’t be omitted for the following functions:
|
||||
|
||||
- [arrayMap](#higher_order_functions-array-map)
|
||||
- [arrayFilter](#higher_order_functions-array-filter)
|
||||
- [arrayFill](#higher_order_functions-array-fill)
|
||||
- [arrayReverseFill](#higher_order_functions-array-reverse-fill)
|
||||
- [arraySplit](#higher_order_functions-array-split)
|
||||
- [arrayReverseSplit](#higher_order_functions-array-reverse-split)
|
||||
- [arrayFirst](#higher_order_functions-array-first)
|
||||
- [arrayFirstIndex](#higher_order_functions-array-first-index)
|
||||
|
||||
### arrayMap(func, arr1, …) {#higher_order_functions-array-map}
|
||||
|
||||
Returns an array obtained from the original application of the `func` function to each element in the `arr` array.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────┐
|
||||
│ [3,4,5] │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
The following example shows how to create a tuple of elements from different arrays:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────────┐
|
||||
│ [(1,4),(2,5),(3,6)] │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayMap` function.
|
||||
|
||||
### arrayFilter(func, arr1, …) {#higher_order_functions-array-filter}
|
||||
|
||||
Returns an array containing only the elements in `arr1` for which `func` returns something other than 0.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ ['abc World'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
arrayFilter(
|
||||
(i, x) -> x LIKE '%World%',
|
||||
arrayEnumerate(arr),
|
||||
['Hello', 'abc World'] AS arr)
|
||||
AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─┐
|
||||
│ [2] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayFilter` function.
|
||||
|
||||
### arrayFill(func, arr1, …) {#higher_order_functions-array-fill}
|
||||
|
||||
Scan through `arr1` from the first element to the last element and replace `arr1[i]` by `arr1[i - 1]` if `func` returns 0. The first element of `arr1` will not be replaced.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────────────────────────┐
|
||||
│ [1,1,3,11,12,12,12,5,6,14,14,14] │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayFill` function.
|
||||
|
||||
### arrayReverseFill(func, arr1, …) {#higher_order_functions-array-reverse-fill}
|
||||
|
||||
Scan through `arr1` from the last element to the first element and replace `arr1[i]` by `arr1[i + 1]` if `func` returns 0. The last element of `arr1` will not be replaced.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res────────────────────────────────┐
|
||||
│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │
|
||||
└────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayReverseFill` function.
|
||||
|
||||
### arraySplit(func, arr1, …) {#higher_order_functions-array-split}
|
||||
|
||||
Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the left hand side of the element. The array will not be split before the first element.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────┐
|
||||
│ [[1,2,3],[4,5]] │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arraySplit` function.
|
||||
|
||||
### arrayReverseSplit(func, arr1, …) {#higher_order_functions-array-reverse-split}
|
||||
|
||||
Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the right hand side of the element. The array will not be split after the last element.
|
||||
|
||||
Examples:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────────┐
|
||||
│ [[1],[2,3,4],[5]] │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arraySplit` function.
|
||||
|
||||
### arrayCount(\[func,\] arr1, …) {#higher_order_functions-array-count}
|
||||
|
||||
Returns the number of elements in the arr array for which func returns something other than 0. If ‘func’ is not specified, it returns the number of non-zero elements in the array.
|
||||
|
||||
### arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1}
|
||||
|
||||
Returns 1 if there is at least one element in ‘arr’ for which ‘func’ returns something other than 0. Otherwise, it returns 0.
|
||||
|
||||
### arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1}
|
||||
|
||||
Returns 1 if ‘func’ returns something other than 0 for all the elements in ‘arr’. Otherwise, it returns 0.
|
||||
|
||||
### arraySum(\[func,\] arr1, …) {#higher-order-functions-array-sum}
|
||||
|
||||
Returns the sum of the ‘func’ values. If the function is omitted, it just returns the sum of the array elements.
|
||||
|
||||
### arrayFirst(func, arr1, …) {#higher_order_functions-array-first}
|
||||
|
||||
Returns the first element in the ‘arr1’ array for which ‘func’ returns something other than 0.
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayFirst` function.
|
||||
|
||||
### arrayFirstIndex(func, arr1, …) {#higher_order_functions-array-first-index}
|
||||
|
||||
Returns the index of the first element in the ‘arr1’ array for which ‘func’ returns something other than 0.
|
||||
|
||||
Note that the first argument (lambda function) can’t be omitted in the `arrayFirstIndex` function.
|
||||
|
||||
### arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1}
|
||||
|
||||
Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSum([1, 1, 1, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────┐
|
||||
│ [1, 2, 3, 4] │
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
### arrayCumSumNonNegative(arr) {#arraycumsumnonnegativearr}
|
||||
|
||||
Same as `arrayCumSum`, returns an array of partial sums of elements in the source array (a running sum). Different `arrayCumSum`, when then returned value contains a value less than zero, the value is replace with zero and the subsequent calculation is performed with zero parameters. For example:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────┐
|
||||
│ [1,2,0,1] │
|
||||
└───────────┘
|
||||
```
|
||||
|
||||
### arraySort(\[func,\] arr1, …) {#arraysortfunc-arr1}
|
||||
|
||||
Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)
|
||||
|
||||
The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to improve sorting efficiency.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]);
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res────────────────┐
|
||||
│ ['world', 'hello'] │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
For more information about the `arraySort` method, see the [Functions for Working With Arrays](../../sql-reference/functions/array-functions.md#array_functions-sort) section.
|
||||
|
||||
### arrayReverseSort(\[func,\] arr1, …) {#arrayreversesortfunc-arr1}
|
||||
|
||||
Returns an array as result of sorting the elements of `arr1` in descending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays).
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────────┐
|
||||
│ ['hello','world'] │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
For more information about the `arrayReverseSort` method, see the [Functions for Working With Arrays](../../sql-reference/functions/array-functions.md#array_functions-reverse-sort) section.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/query_language/functions/higher_order_functions/) <!--hide-->
|
||||
@ -44,6 +44,21 @@ Functions have the following behaviors:
|
||||
|
||||
Functions can’t change the values of their arguments – any changes are returned as the result. Thus, the result of calculating separate functions does not depend on the order in which the functions are written in the query.
|
||||
|
||||
## Higher-order functions, `->` operator and lambda(params, expr) function {#higher-order-functions}
|
||||
|
||||
Higher-order functions can only accept lambda functions as their functional argument. To pass a lambda function to a higher-order function use `->` operator. The left side of the arrow has a formal parameter, which is any ID, or multiple formal parameters – any IDs in a tuple. The right side of the arrow has an expression that can use these formal parameters, as well as any table columns.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
x -> 2 * x
|
||||
str -> str != Referer
|
||||
```
|
||||
|
||||
A lambda function that accepts multiple arguments can also be passed to a higher-order function. In this case, the higher-order function is passed several arrays of identical length that these arguments will correspond to.
|
||||
|
||||
For some functions the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed.
|
||||
|
||||
## Error Handling {#error-handling}
|
||||
|
||||
Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query.
|
||||
|
||||
@ -98,7 +98,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
The [arrayMap](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `addressToLine` function. The result of this processing you see in the `trace_source_code_lines` column of output.
|
||||
The [arrayMap](../../sql-reference/functions/array-functions.md#array-map) function allows to process each individual element of the `trace` array by the `addressToLine` function. The result of this processing you see in the `trace_source_code_lines` column of output.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
@ -184,7 +184,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
The [arrayMap](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `addressToSymbols` function. The result of this processing you see in the `trace_symbols` column of output.
|
||||
The [arrayMap](../../sql-reference/functions/array-functions.md#array-map) function allows to process each individual element of the `trace` array by the `addressToSymbols` function. The result of this processing you see in the `trace_symbols` column of output.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
@ -281,7 +281,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
The [arrayMap](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `demangle` function. The result of this processing you see in the `trace_functions` column of output.
|
||||
The [arrayMap](../../sql-reference/functions/array-functions.md#array-map) function allows to process each individual element of the `trace` array by the `demangle` function. The result of this processing you see in the `trace_functions` column of output.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
|
||||
@ -1,3 +1,8 @@
|
||||
---
|
||||
toc_priority: 30
|
||||
toc_title: MergeTree
|
||||
---
|
||||
|
||||
# MergeTree {#table_engines-mergetree}
|
||||
|
||||
Движок `MergeTree`, а также другие движки этого семейства (`*MergeTree`) — это наиболее функциональные движки таблиц ClickHouse.
|
||||
@ -28,8 +33,8 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1,
|
||||
INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY expr
|
||||
[PARTITION BY expr]
|
||||
[ORDER BY expr]
|
||||
[PRIMARY KEY expr]
|
||||
[SAMPLE BY expr]
|
||||
[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...]
|
||||
@ -38,27 +43,42 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
Описание параметров смотрите в [описании запроса CREATE](../../../engines/table-engines/mergetree-family/mergetree.md).
|
||||
|
||||
!!! note "Note"
|
||||
!!! note "Примечание"
|
||||
`INDEX` — экспериментальная возможность, смотрите [Индексы пропуска данных](#table_engine-mergetree-data_skipping-indexes).
|
||||
|
||||
### Секции запроса {#mergetree-query-clauses}
|
||||
|
||||
- `ENGINE` — имя и параметры движка. `ENGINE = MergeTree()`. `MergeTree` не имеет параметров.
|
||||
|
||||
- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table-engines/mergetree-family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`.
|
||||
- `ORDER BY` — ключ сортировки.
|
||||
|
||||
Кортеж столбцов или произвольных выражений. Пример: `ORDER BY (CounterID, EventDate)`.
|
||||
|
||||
- `ORDER BY` — ключ сортировки. Кортеж столбцов или произвольных выражений. Пример: `ORDER BY (CounterID, EventDate)`.
|
||||
ClickHouse использует ключ сортировки в качестве первичного ключа, если первичный ключ не задан в секции `PRIMARY KEY`.
|
||||
|
||||
- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki). По умолчанию первичный ключ совпадает с ключом сортировки (который задаётся секцией `ORDER BY`.) Поэтому в большинстве случаев секцию `PRIMARY KEY` отдельно указывать не нужно.
|
||||
Чтобы отключить сортировку, используйте синтаксис `ORDER BY tuple()`. Смотрите [выбор первичного ключа](#vybor-pervichnogo-kliucha).
|
||||
|
||||
- `SAMPLE BY` — выражение для сэмплирования. Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`.
|
||||
- `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Необязательный параметр.
|
||||
|
||||
- `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. Выражение должно возвращать столбец `Date` или `DateTime`. Пример: `TTL date + INTERVAL 1 DAY`.
|
||||
- Тип правила `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` указывает действие, которое будет выполнено с частью, удаление строк (прореживание), перемещение (при выполнении условия для всех строк части) на определённый диск (`TO DISK 'xxx'`) или том (`TO VOLUME 'xxx'`).
|
||||
- Поведение по умолчанию соответствует удалению строк (`DELETE`). В списке правил может быть указано только одно выражение с поведением `DELETE`.
|
||||
- Дополнительные сведения смотрите в разделе [TTL для столбцов и таблиц](#table_engine-mergetree-ttl)
|
||||
Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table-engines/mergetree-family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`.
|
||||
|
||||
- `SETTINGS` — дополнительные параметры, регулирующие поведение `MergeTree`:
|
||||
- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki). Необязательный параметр.
|
||||
|
||||
По умолчанию первичный ключ совпадает с ключом сортировки (который задаётся секцией `ORDER BY`.) Поэтому в большинстве случаев секцию `PRIMARY KEY` отдельно указывать не нужно.
|
||||
|
||||
- `SAMPLE BY` — выражение для сэмплирования. Необязательный параметр.
|
||||
|
||||
Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`.
|
||||
|
||||
- `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. Необязательный параметр.
|
||||
|
||||
Выражение должно возвращать столбец `Date` или `DateTime`. Пример: `TTL date + INTERVAL 1 DAY`.
|
||||
|
||||
Тип правила `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` указывает действие, которое будет выполнено с частью, удаление строк (прореживание), перемещение (при выполнении условия для всех строк части) на определённый диск (`TO DISK 'xxx'`) или том (`TO VOLUME 'xxx'`). Поведение по умолчанию соответствует удалению строк (`DELETE`). В списке правил может быть указано только одно выражение с поведением `DELETE`.
|
||||
|
||||
Дополнительные сведения смотрите в разделе [TTL для столбцов и таблиц](#table_engine-mergetree-ttl)
|
||||
|
||||
- `SETTINGS` — дополнительные параметры, регулирующие поведение `MergeTree` (необязательные):
|
||||
|
||||
- `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage).
|
||||
- `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage).
|
||||
@ -180,6 +200,14 @@ ClickHouse не требует уникального первичного кл
|
||||
|
||||
Длинный первичный ключ будет негативно влиять на производительность вставки и потребление памяти, однако на производительность ClickHouse при запросах `SELECT` лишние столбцы в первичном ключе не влияют.
|
||||
|
||||
Вы можете создать таблицу без первичного ключа, используя синтаксис `ORDER BY tuple()`. В этом случае ClickHouse хранит данные в порядке вставки. Если вы хотите сохранить порядок данных при вставке данных с помощью запросов `INSERT ... SELECT`, установите [max\_insert\_threads = 1](../../../operations/settings/settings.md#settings-max-insert-threads).
|
||||
|
||||
Чтобы выбрать данные в первоначальном порядке, используйте
|
||||
[однопоточные](../../../operations/settings/settings.md#settings-max_threads) запросы `SELECT.
|
||||
|
||||
|
||||
|
||||
|
||||
### Первичный ключ, отличный от ключа сортировки {#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki}
|
||||
|
||||
Существует возможность задать первичный ключ (выражение, значения которого будут записаны в индексный файл для
|
||||
|
||||
4
docs/ru/interfaces/third-party/gui.md
vendored
4
docs/ru/interfaces/third-party/gui.md
vendored
@ -93,6 +93,10 @@
|
||||
|
||||
[cickhouse-plantuml](https://pypi.org/project/clickhouse-plantuml/) — скрипт, генерирующий [PlantUML](https://plantuml.com/) диаграммы схем таблиц.
|
||||
|
||||
### xeus-clickhouse {#xeus-clickhouse}
|
||||
|
||||
[xeus-clickhouse](https://github.com/wangfenjin/xeus-clickhouse) — это ядро Jupyter для ClickHouse, которое поддерживает запрос ClickHouse-данных с использованием SQL в Jupyter.
|
||||
|
||||
## Коммерческие {#kommercheskie}
|
||||
|
||||
### DataGrip {#datagrip}
|
||||
|
||||
@ -7,10 +7,38 @@ toc_title: Системные таблицы
|
||||
|
||||
## Введение {#system-tables-introduction}
|
||||
|
||||
Системные таблицы используются для реализации части функциональности системы, а также предоставляют доступ к информации о работе системы.
|
||||
Вы не можете удалить системную таблицу (хотя можете сделать DETACH).
|
||||
Для системных таблиц нет файлов с данными на диске и файлов с метаданными. Сервер создаёт все системные таблицы при старте.
|
||||
В системные таблицы нельзя записывать данные - можно только читать.
|
||||
Системные таблицы расположены в базе данных system.
|
||||
Системные таблицы содержат информацию о:
|
||||
|
||||
- Состоянии сервера, процессов и окружении.
|
||||
- Внутренних процессах сервера.
|
||||
|
||||
Системные таблицы:
|
||||
|
||||
- Находятся в базе данных `system`.
|
||||
- Доступны только для чтения данных.
|
||||
- Не могут быть удалены или изменены, но их можно отсоединить.
|
||||
|
||||
Системные таблицы `metric_log`, `query_log`, `query_thread_log`, `trace_log` системные таблицы хранят данные в файловой системе. Остальные системные таблицы хранят свои данные в оперативной памяти. Сервер ClickHouse создает такие системные таблицы при запуске.
|
||||
|
||||
### Источники системных показателей
|
||||
|
||||
Для сбора системных показателей сервер ClickHouse использует:
|
||||
|
||||
- Возможности `CAP_NET_ADMIN`.
|
||||
- [procfs](https://ru.wikipedia.org/wiki/Procfs) (только Linux).
|
||||
|
||||
**procfs**
|
||||
|
||||
Если для сервера ClickHouse не включено `CAP_NET_ADMIN`, он пытается обратиться к `ProcfsMetricsProvider`. `ProcfsMetricsProvider` позволяет собирать системные показатели для каждого запроса (для CPU и I/O).
|
||||
|
||||
Если procfs поддерживается и включена в системе, то сервер ClickHouse собирает следующие системные показатели:
|
||||
|
||||
- `OSCPUVirtualTimeMicroseconds`
|
||||
- `OSCPUWaitMicroseconds`
|
||||
- `OSIOWaitMicroseconds`
|
||||
- `OSReadChars`
|
||||
- `OSWriteChars`
|
||||
- `OSReadBytes`
|
||||
- `OSWriteBytes`
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system-tables/) <!--hide-->
|
||||
|
||||
@ -82,7 +82,7 @@ res: /lib/x86_64-linux-gnu/libc-2.27.so
|
||||
|
||||
- [Функции интроспекции](../../sql-reference/functions/introspection.md) — Что такое функции интроспекции и как их использовать.
|
||||
- [system.trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) — Содержит трассировки стека, собранные профилировщиком выборочных запросов.
|
||||
- [arrayMap](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-map) — Описание и пример использования функции `arrayMap`.
|
||||
- [arrayFilter](../../sql-reference/functions/higher-order-functions.md#higher_order_functions-array-filter) — Описание и пример использования функции `arrayFilter`.
|
||||
- [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — Описание и пример использования функции `arrayMap`.
|
||||
- [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — Описание и пример использования функции `arrayFilter`.
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/stack_trace) <!--hide-->
|
||||
|
||||
@ -9,6 +9,7 @@ The following aggregate functions are supported:
|
||||
- [`min`](../../sql-reference/aggregate-functions/reference/min.md#agg_function-min)
|
||||
- [`max`](../../sql-reference/aggregate-functions/reference/max.md#agg_function-max)
|
||||
- [`sum`](../../sql-reference/aggregate-functions/reference/sum.md#agg_function-sum)
|
||||
- [`sumWithOverflow`](../../sql-reference/aggregate-functions/reference/sumwithoverflow.md#sumwithoverflowx)
|
||||
- [`groupBitAnd`](../../sql-reference/aggregate-functions/reference/groupbitand.md#groupbitand)
|
||||
- [`groupBitOr`](../../sql-reference/aggregate-functions/reference/groupbitor.md#groupbitor)
|
||||
- [`groupBitXor`](../../sql-reference/aggregate-functions/reference/groupbitxor.md#groupbitxor)
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
Кортеж из элементов любого [типа](index.md#data_types). Элементы кортежа могут быть одного или разных типов.
|
||||
|
||||
Кортежи используются для временной группировки столбцов. Столбцы могут группироваться при использовании выражения IN в запросе, а также для указания нескольких формальных параметров лямбда-функций. Подробнее смотрите разделы [Операторы IN](../../sql-reference/data-types/tuple.md), [Функции высшего порядка](../../sql-reference/functions/higher-order-functions.md#higher-order-functions).
|
||||
Кортежи используются для временной группировки столбцов. Столбцы могут группироваться при использовании выражения IN в запросе, а также для указания нескольких формальных параметров лямбда-функций. Подробнее смотрите разделы [Операторы IN](../../sql-reference/data-types/tuple.md), [Функции высшего порядка](../../sql-reference/functions/index.md#higher-order-functions).
|
||||
|
||||
Кортежи могут быть результатом запроса. В этом случае, в текстовых форматах кроме JSON, значения выводятся в круглых скобках через запятую. В форматах JSON, кортежи выводятся в виде массивов (в квадратных скобках).
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# Функции по работе с массивами {#funktsii-po-rabote-s-massivami}
|
||||
# Массивы {#functions-for-working-with-arrays}
|
||||
|
||||
## empty {#function-empty}
|
||||
|
||||
@ -186,6 +186,13 @@ SELECT indexOf([1, 3, NULL, NULL], NULL)
|
||||
|
||||
Элементы, равные `NULL`, обрабатываются как обычные значения.
|
||||
|
||||
## arrayCount(\[func,\] arr1, …) {#array-count}
|
||||
|
||||
Возвращает количество элементов массива `arr`, для которых функция `func` возвращает не 0. Если `func` не указана - возвращает количество ненулевых элементов массива.
|
||||
|
||||
Функция `arrayCount` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей можно передать лямбда-функцию.
|
||||
|
||||
|
||||
## countEqual(arr, x) {#countequalarr-x}
|
||||
|
||||
Возвращает количество элементов массива, равных x. Эквивалентно arrayCount(elem -\> elem = x, arr).
|
||||
@ -513,7 +520,7 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
|
||||
- Значения `NaN` идут перед `NULL`.
|
||||
- Значения `Inf` идут перед `NaN`.
|
||||
|
||||
Функция `arraySort` является [функцией высшего порядка](higher-order-functions.md) — в качестве первого аргумента ей можно передать лямбда-функцию. В этом случае порядок сортировки определяется результатом применения лямбда-функции на элементы массива.
|
||||
Функция `arraySort` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей можно передать лямбда-функцию. В этом случае порядок сортировки определяется результатом применения лямбда-функции на элементы массива.
|
||||
|
||||
Рассмотрим пример:
|
||||
|
||||
@ -613,7 +620,7 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
|
||||
- Значения `NaN` идут перед `NULL`.
|
||||
- Значения `-Inf` идут перед `NaN`.
|
||||
|
||||
Функция `arrayReverseSort` является [функцией высшего порядка](higher-order-functions.md). Вы можете передать ей в качестве первого аргумента лямбда-функцию. Например:
|
||||
Функция `arrayReverseSort` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей можно передать лямбда-функцию. Например:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
|
||||
@ -1036,6 +1043,116 @@ SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1])
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## arrayMap(func, arr1, …) {#array-map}
|
||||
|
||||
Возвращает массив, полученный на основе результатов применения функции `func` к каждому элементу массива `arr`.
|
||||
|
||||
Примеры:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────┐
|
||||
│ [3,4,5] │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Следующий пример показывает, как создать кортежи из элементов разных массивов:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────────┐
|
||||
│ [(1,4),(2,5),(3,6)] │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
Функция `arrayMap` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен.
|
||||
|
||||
## arrayFilter(func, arr1, …) {#array-filter}
|
||||
|
||||
Возвращает массив, содержащий только те элементы массива `arr1`, для которых функция `func` возвращает не 0.
|
||||
|
||||
Примеры:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ ['abc World'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
arrayFilter(
|
||||
(i, x) -> x LIKE '%World%',
|
||||
arrayEnumerate(arr),
|
||||
['Hello', 'abc World'] AS arr)
|
||||
AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─┐
|
||||
│ [2] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
Функция `arrayFilter` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен.
|
||||
|
||||
## arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1}
|
||||
|
||||
Возвращает 1, если существует хотя бы один элемент массива `arr`, для которого функция func возвращает не 0. Иначе возвращает 0.
|
||||
|
||||
Функция `arrayExists` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) - в качестве первого аргумента ей можно передать лямбда-функцию.
|
||||
|
||||
## arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1}
|
||||
|
||||
Возвращает 1, если для всех элементов массива `arr`, функция `func` возвращает не 0. Иначе возвращает 0.
|
||||
|
||||
Функция `arrayAll` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) - в качестве первого аргумента ей можно передать лямбда-функцию.
|
||||
|
||||
## arrayFirst(func, arr1, …) {#array-first}
|
||||
|
||||
Возвращает первый элемент массива `arr1`, для которого функция func возвращает не 0.
|
||||
|
||||
Функция `arrayFirst` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен.
|
||||
|
||||
## arrayFirstIndex(func, arr1, …) {#array-first-index}
|
||||
|
||||
Возвращает индекс первого элемента массива `arr1`, для которого функция func возвращает не 0.
|
||||
|
||||
Функция `arrayFirstIndex` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен.
|
||||
|
||||
## arraySum(\[func,\] arr1, …) {#array-sum}
|
||||
|
||||
Возвращает сумму значений функции `func`. Если функция не указана - просто возвращает сумму элементов массива.
|
||||
|
||||
Функция `arraySum` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) - в качестве первого аргумента ей можно передать лямбда-функцию.
|
||||
|
||||
## arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1}
|
||||
|
||||
Возвращает массив из частичных сумм элементов исходного массива (сумма с накоплением). Если указана функция `func`, то значения элементов массива преобразуются этой функцией перед суммированием.
|
||||
|
||||
Функция `arrayCumSum` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) - в качестве первого аргумента ей можно передать лямбда-функцию.
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSum([1, 1, 1, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────┐
|
||||
│ [1, 2, 3, 4] │
|
||||
└──────────────┘
|
||||
|
||||
## arrayAUC {#arrayauc}
|
||||
|
||||
Вычисляет площадь под кривой.
|
||||
|
||||
@ -1,167 +0,0 @@
|
||||
# Функции высшего порядка {#higher-order-functions}
|
||||
|
||||
## Оператор `->`, функция lambda(params, expr) {#operator-funktsiia-lambdaparams-expr}
|
||||
|
||||
Позволяет описать лямбда-функцию для передачи в функцию высшего порядка. Слева от стрелочки стоит формальный параметр - произвольный идентификатор, или несколько формальных параметров - произвольные идентификаторы в кортеже. Справа от стрелочки стоит выражение, в котором могут использоваться эти формальные параметры, а также любые столбцы таблицы.
|
||||
|
||||
Примеры: `x -> 2 * x, str -> str != Referer.`
|
||||
|
||||
Функции высшего порядка, в качестве своего функционального аргумента могут принимать только лямбда-функции.
|
||||
|
||||
В функции высшего порядка может быть передана лямбда-функция, принимающая несколько аргументов. В этом случае, в функцию высшего порядка передаётся несколько массивов одинаковых длин, которым эти аргументы будут соответствовать.
|
||||
|
||||
Для некоторых функций, например [arrayCount](#higher_order_functions-array-count) или [arraySum](#higher_order_functions-array-sum), первый аргумент (лямбда-функция) может отсутствовать. В этом случае, подразумевается тождественное отображение.
|
||||
|
||||
Для функций, перечисленных ниже, лямбда-функцию должна быть указана всегда:
|
||||
|
||||
- [arrayMap](#higher_order_functions-array-map)
|
||||
- [arrayFilter](#higher_order_functions-array-filter)
|
||||
- [arrayFirst](#higher_order_functions-array-first)
|
||||
- [arrayFirstIndex](#higher_order_functions-array-first-index)
|
||||
|
||||
### arrayMap(func, arr1, …) {#higher_order_functions-array-map}
|
||||
|
||||
Вернуть массив, полученный на основе результатов применения функции `func` к каждому элементу массива `arr`.
|
||||
|
||||
Примеры:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────┐
|
||||
│ [3,4,5] │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Следующий пример показывает, как создать кортежи из элементов разных массивов:
|
||||
|
||||
``` sql
|
||||
SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─────────────────┐
|
||||
│ [(1,4),(2,5),(3,6)] │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
Обратите внимание, что у функции `arrayMap` первый аргумент (лямбда-функция) не может быть опущен.
|
||||
|
||||
### arrayFilter(func, arr1, …) {#higher_order_functions-array-filter}
|
||||
|
||||
Вернуть массив, содержащий только те элементы массива `arr1`, для которых функция `func` возвращает не 0.
|
||||
|
||||
Примеры:
|
||||
|
||||
``` sql
|
||||
SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ ['abc World'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
arrayFilter(
|
||||
(i, x) -> x LIKE '%World%',
|
||||
arrayEnumerate(arr),
|
||||
['Hello', 'abc World'] AS arr)
|
||||
AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res─┐
|
||||
│ [2] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
Обратите внимание, что у функции `arrayFilter` первый аргумент (лямбда-функция) не может быть опущен.
|
||||
|
||||
### arrayCount(\[func,\] arr1, …) {#higher_order_functions-array-count}
|
||||
|
||||
Вернуть количество элементов массива `arr`, для которых функция func возвращает не 0. Если func не указана - вернуть количество ненулевых элементов массива.
|
||||
|
||||
### arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1}
|
||||
|
||||
Вернуть 1, если существует хотя бы один элемент массива `arr`, для которого функция func возвращает не 0. Иначе вернуть 0.
|
||||
|
||||
### arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1}
|
||||
|
||||
Вернуть 1, если для всех элементов массива `arr`, функция `func` возвращает не 0. Иначе вернуть 0.
|
||||
|
||||
### arraySum(\[func,\] arr1, …) {#higher_order_functions-array-sum}
|
||||
|
||||
Вернуть сумму значений функции `func`. Если функция не указана - просто вернуть сумму элементов массива.
|
||||
|
||||
### arrayFirst(func, arr1, …) {#higher_order_functions-array-first}
|
||||
|
||||
Вернуть первый элемент массива `arr1`, для которого функция func возвращает не 0.
|
||||
|
||||
Обратите внимание, что у функции `arrayFirst` первый аргумент (лямбда-функция) не может быть опущен.
|
||||
|
||||
### arrayFirstIndex(func, arr1, …) {#higher_order_functions-array-first-index}
|
||||
|
||||
Вернуть индекс первого элемента массива `arr1`, для которого функция func возвращает не 0.
|
||||
|
||||
Обратите внимание, что у функции `arrayFirstFilter` первый аргумент (лямбда-функция) не может быть опущен.
|
||||
|
||||
### arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1}
|
||||
|
||||
Возвращает массив из частичных сумм элементов исходного массива (сумма с накоплением). Если указана функция `func`, то значения элементов массива преобразуются этой функцией перед суммированием.
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
SELECT arrayCumSum([1, 1, 1, 1]) AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res──────────┐
|
||||
│ [1, 2, 3, 4] │
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
### arraySort(\[func,\] arr1, …) {#arraysortfunc-arr1}
|
||||
|
||||
Возвращает отсортированный в восходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).
|
||||
|
||||
Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]);
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res────────────────┐
|
||||
│ ['world', 'hello'] │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
Подробная информация о методе `arraySort` приведена в разделе [Функции по работе с массивами](array-functions.md#array_functions-sort).
|
||||
|
||||
### arrayReverseSort(\[func,\] arr1, …) {#arrayreversesortfunc-arr1}
|
||||
|
||||
Возвращает отсортированный в нисходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────────┐
|
||||
│ ['hello','world'] │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
Подробная информация о методе `arrayReverseSort` приведена в разделе [Функции по работе с массивами](array-functions.md#array_functions-reverse-sort).
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/higher_order_functions/) <!--hide-->
|
||||
@ -38,6 +38,20 @@
|
||||
|
||||
Функции не могут поменять значения своих аргументов - любые изменения возвращаются в качестве результата. Соответственно, от порядка записи функций в запросе, результат вычислений отдельных функций не зависит.
|
||||
|
||||
## Функции высшего порядка, оператор `->` и функция lambda(params, expr) {#higher-order-functions}
|
||||
|
||||
Функции высшего порядка, в качестве своего функционального аргумента могут принимать только лямбда-функции. Чтобы передать лямбда-функцию в функцию высшего порядка, используйте оператор `->`. Слева от стрелочки стоит формальный параметр — произвольный идентификатор, или несколько формальных параметров — произвольные идентификаторы в кортеже. Справа от стрелочки стоит выражение, в котором могут использоваться эти формальные параметры, а также любые столбцы таблицы.
|
||||
|
||||
Примеры:
|
||||
```
|
||||
x -> 2 * x
|
||||
str -> str != Referer
|
||||
```
|
||||
|
||||
В функции высшего порядка может быть передана лямбда-функция, принимающая несколько аргументов. В этом случае в функцию высшего порядка передаётся несколько массивов одинаковой длины, которым эти аргументы будут соответствовать.
|
||||
|
||||
Для некоторых функций первый аргумент (лямбда-функция) может отсутствовать. В этом случае подразумевается тождественное отображение.
|
||||
|
||||
## Обработка ошибок {#obrabotka-oshibok}
|
||||
|
||||
Некоторые функции могут кидать исключения в случае ошибочных данных. В этом случае, выполнение запроса прерывается, и текст ошибки выводится клиенту. При распределённой обработке запроса, при возникновении исключения на одном из серверов, на другие серверы пытается отправиться просьба тоже прервать выполнение запроса.
|
||||
|
||||
@ -93,7 +93,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
Функция [arrayMap](higher-order-functions.md#higher_order_functions-array-map) позволяет обрабатывать каждый отдельный элемент массива `trace` с помощью функции `addressToLine`. Результат этой обработки вы видите в виде `trace_source_code_lines` колонки выходных данных.
|
||||
Функция [arrayMap](../../sql-reference/functions/array-functions.md#array-map) позволяет обрабатывать каждый отдельный элемент массива `trace` с помощью функции `addressToLine`. Результат этой обработки вы видите в виде `trace_source_code_lines` колонки выходных данных.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
@ -179,7 +179,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
То [arrayMap](higher-order-functions.md#higher_order_functions-array-map) функция позволяет обрабатывать каждый отдельный элемент системы. `trace` массив по типу `addressToSymbols` функция. Результат этой обработки вы видите в виде `trace_symbols` колонка выходных данных.
|
||||
То [arrayMap](../../sql-reference/functions/array-functions.md#array-map) функция позволяет обрабатывать каждый отдельный элемент системы. `trace` массив по типу `addressToSymbols` функция. Результат этой обработки вы видите в виде `trace_symbols` колонка выходных данных.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
@ -276,7 +276,7 @@ LIMIT 1
|
||||
\G
|
||||
```
|
||||
|
||||
Функция [arrayMap](higher-order-functions.md#higher_order_functions-array-map) позволяет обрабатывать каждый отдельный элемент массива `trace` с помощью функции `demangle`.
|
||||
Функция [arrayMap](../../sql-reference/functions/array-functions.md#array-map) позволяет обрабатывать каждый отдельный элемент массива `trace` с помощью функции `demangle`.
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
|
||||
@ -1,6 +1,4 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 12
|
||||
toc_title: "\u6559\u7A0B"
|
||||
---
|
||||
@ -9,27 +7,27 @@ toc_title: "\u6559\u7A0B"
|
||||
|
||||
## 从本教程中可以期待什么？ {#what-to-expect-from-this-tutorial}
|
||||
|
||||
通过本教程，您将学习如何设置一个简单的ClickHouse集群。 它会很小，但容错和可扩展。 然后，我们将使用其中一个示例数据集来填充数据并执行一些演示查询。
|
||||
通过本教程，您将学习如何设置一个简单的ClickHouse集群。 它会很小，但却是容错和可扩展的。 然后，我们将使用其中一个示例数据集来填充数据并执行一些演示查询。
|
||||
|
||||
## 单节点设置 {#single-node-setup}
|
||||
|
||||
为了推迟分布式环境的复杂性，我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从安装 [黛布](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包，但也有 [替代办法](install.md#from-docker-image) 对于不支持它们的操作系统。
|
||||
为了推迟分布式环境的复杂性，我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从[deb](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包安装，但对于不支持它们的操作系统也有 [替代方法](install.md#from-docker-image) 。
|
||||
|
||||
例如，您选择了 `deb` 包和执行:
|
||||
例如，您选择了从 `deb` 包安装，执行:
|
||||
|
||||
``` bash
|
||||
{% include 'install/deb.sh' %}
|
||||
```
|
||||
|
||||
我们在安装的软件包中有什么:
|
||||
在我们安装的软件中包含这些包:
|
||||
|
||||
- `clickhouse-client` 包包含 [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../interfaces/cli.md) 应用程序，交互式ClickHouse控制台客户端。
|
||||
- `clickhouse-common` 包包含一个ClickHouse可执行文件。
|
||||
- `clickhouse-server` 包包含要作为服务器运行ClickHouse的配置文件。
|
||||
- `clickhouse-client` 包，包含 [clickhouse-client](../interfaces/cli.md) 应用程序，它是交互式ClickHouse控制台客户端。
|
||||
- `clickhouse-common` 包，包含一个ClickHouse可执行文件。
|
||||
- `clickhouse-server` 包，包含要作为服务端运行的ClickHouse配置文件。
|
||||
|
||||
服务器配置文件位于 `/etc/clickhouse-server/`. 在进一步讨论之前，请注意 `<path>` 元素in `config.xml`. Path确定数据存储的位置，因此应该位于磁盘容量较大的卷上；默认值为 `/var/lib/clickhouse/`. 如果你想调整配置，直接编辑并不方便 `config.xml` 文件，考虑到它可能会在未来的软件包更新中被重写。 复盖配置元素的推荐方法是创建 [在配置文件。d目录](../operations/configuration-files.md) 它作为 “patches” 要配置。xml
|
||||
服务端配置文件位于 `/etc/clickhouse-server/`。 在进一步讨论之前，请注意 `config.xml`文件中的`<path>` 元素. Path决定了数据存储的位置，因此该位置应该位于磁盘容量较大的卷上；默认值为 `/var/lib/clickhouse/`。 如果你想调整配置，考虑到它可能会在未来的软件包更新中被重写，直接编辑`config.xml` 文件并不方便。 推荐的方法是在[配置文件](../operations/configuration-files.md)目录创建文件，作为config.xml文件的“补丁”，用以复写配置元素。
|
||||
|
||||
你可能已经注意到了, `clickhouse-server` 安装包后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务器的方式取决于您的init系统，通常情况下，它是:
|
||||
你可能已经注意到了, `clickhouse-server` 安装后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务端的方式取决于您的初始系统，通常情况下是这样:
|
||||
|
||||
``` bash
|
||||
sudo service clickhouse-server start
|
||||
@ -41,13 +39,13 @@ sudo service clickhouse-server start
|
||||
sudo /etc/init.d/clickhouse-server start
|
||||
```
|
||||
|
||||
服务器日志的默认位置是 `/var/log/clickhouse-server/`. 服务器已准备好处理客户端连接一旦它记录 `Ready for connections` 消息
|
||||
服务端日志的默认位置是 `/var/log/clickhouse-server/`。当服务端在日志中记录 `Ready for connections` 消息，即表示服务端已准备好处理客户端连接。
|
||||
|
||||
一旦 `clickhouse-server` 正在运行我们可以利用 `clickhouse-client` 连接到服务器并运行一些测试查询，如 `SELECT "Hello, world!";`.
|
||||
一旦 `clickhouse-server` 启动并运行，我们可以利用 `clickhouse-client` 连接到服务端，并运行一些测试查询，如 `SELECT "Hello, world!";`.
|
||||
|
||||
<details markdown="1">
|
||||
|
||||
<summary>Clickhouse-客户端的快速提示</summary>
|
||||
<summary>Clickhouse-client的快速提示</summary>
|
||||
|
||||
交互模式:
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ toc_title: "\u5E94\u7528CatBoost\u6A21\u578B"
|
||||
|
||||
1. [创建表](#create-table).
|
||||
2. [将数据插入到表中](#insert-data-to-table).
|
||||
3. [碌莽禄into拢Integrate010-68520682\<url\>](#integrate-catboost-into-clickhouse) （可选步骤）。
|
||||
3. [将CatBoost集成到ClickHouse中](#integrate-catboost-into-clickhouse) （可选步骤）。
|
||||
4. [从SQL运行模型推理](#run-model-inference).
|
||||
|
||||
有关训练CatBoost模型的详细信息，请参阅 [培训和应用模型](https://catboost.ai/docs/features/training.html#training).
|
||||
@ -119,12 +119,12 @@ FROM amazon_train
|
||||
+-------+
|
||||
```
|
||||
|
||||
## 3. 碌莽禄into拢Integrate010-68520682\<url\> {#integrate-catboost-into-clickhouse}
|
||||
## 3. 将CatBoost集成到ClickHouse中 {#integrate-catboost-into-clickhouse}
|
||||
|
||||
!!! note "注"
|
||||
**可选步骤。** Docker映像包含运行CatBoost和ClickHouse所需的所有内容。
|
||||
|
||||
碌莽禄to拢integrate010-68520682\<url\>:
|
||||
CatBoost集成到ClickHouse步骤:
|
||||
|
||||
**1.** 构建评估库。
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
|
||||
@ -86,6 +87,8 @@ namespace
|
||||
case ValueType::vtUUID:
|
||||
assert_cast<ColumnUInt128 &>(column).insert(parse<UUID>(value.convert<std::string>()));
|
||||
break;
|
||||
default:
|
||||
throw Exception("Unsupported value type", ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -13,6 +13,11 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using ValueType = ExternalResultDescription::ValueType;
|
||||
@ -79,6 +84,9 @@ namespace
|
||||
return Poco::Dynamic::Var(std::to_string(LocalDateTime(time_t(field.get<UInt64>())))).convert<String>();
|
||||
case ValueType::vtUUID:
|
||||
return Poco::Dynamic::Var(UUID(field.get<UInt128>()).toUnderType().toHexString()).convert<std::string>();
|
||||
default:
|
||||
throw Exception("Unsupported value type", ErrorCodes::UNKNOWN_TYPE);
|
||||
|
||||
}
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
@ -12,6 +12,9 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorArray final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -45,6 +48,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorArray>());
|
||||
|
||||
@ -6,12 +6,14 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorDistinct final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -56,6 +58,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorDistinct(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorDistinct>());
|
||||
|
||||
@ -12,6 +12,9 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorForEach final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -42,6 +45,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorForEach(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorForEach>());
|
||||
|
||||
@ -13,6 +13,9 @@ namespace ErrorCodes
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorMerge final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -55,6 +58,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorMerge>());
|
||||
|
||||
@ -15,6 +15,9 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorNull final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -119,6 +122,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorNull>());
|
||||
|
||||
@ -6,6 +6,8 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace
|
||||
{
|
||||
|
||||
template <bool UseNull>
|
||||
class AggregateFunctionCombinatorOrFill final : public IAggregateFunctionCombinator
|
||||
@ -32,6 +34,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorOrFill(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorOrFill<false>>());
|
||||
|
||||
@ -13,6 +13,9 @@ namespace ErrorCodes
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorResample final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -93,6 +96,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorResample(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorResample>());
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <common/arithmeticOverflow.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -60,7 +61,18 @@ public:
|
||||
if (end < begin)
|
||||
total = 0;
|
||||
else
|
||||
total = (end - begin + step - 1) / step;
|
||||
{
|
||||
Key dif;
|
||||
size_t sum;
|
||||
if (common::subOverflow(end, begin, dif)
|
||||
|| common::addOverflow(static_cast<size_t>(dif), step, sum))
|
||||
{
|
||||
throw Exception("Overflow in internal computations in function " + getName()
|
||||
+ ". Too large arguments", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
}
|
||||
|
||||
total = (sum - 1) / step; // total = (end - begin + step - 1) / step
|
||||
}
|
||||
|
||||
if (total > MAX_ELEMENTS)
|
||||
throw Exception("The range given in function "
|
||||
|
||||
@ -13,6 +13,9 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class AggregateFunctionCombinatorState final : public IAggregateFunctionCombinator
|
||||
{
|
||||
public:
|
||||
@ -33,6 +36,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionCombinatorState(AggregateFunctionCombinatorFactory & factory)
|
||||
{
|
||||
factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorState>());
|
||||
|
||||
@ -170,7 +170,6 @@ public:
|
||||
size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); }
|
||||
bool isNumeric() const override { return getDictionary().isNumeric(); }
|
||||
bool lowCardinality() const override { return true; }
|
||||
bool isNullable() const override { return isColumnNullable(*dictionary.getColumnUniquePtr()); }
|
||||
|
||||
const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); }
|
||||
const ColumnPtr & getDictionaryPtr() const { return dictionary.getColumnUniquePtr(); }
|
||||
|
||||
@ -281,7 +281,7 @@ namespace ErrorCodes
|
||||
extern const int DICTIONARY_IS_EMPTY = 281;
|
||||
extern const int INCORRECT_INDEX = 282;
|
||||
extern const int UNKNOWN_DISTRIBUTED_PRODUCT_MODE = 283;
|
||||
extern const int UNKNOWN_GLOBAL_SUBQUERIES_METHOD = 284;
|
||||
extern const int WRONG_GLOBAL_SUBQUERY = 284;
|
||||
extern const int TOO_FEW_LIVE_REPLICAS = 285;
|
||||
extern const int UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE = 286;
|
||||
extern const int UNKNOWN_FORMAT_VERSION = 287;
|
||||
@ -507,6 +507,7 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_DECLARE_RABBITMQ_EXCHANGE = 540;
|
||||
extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING = 541;
|
||||
extern const int CANNOT_REMOVE_RABBITMQ_EXCHANGE = 542;
|
||||
extern const int UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL = 543;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
||||
@ -72,9 +72,10 @@ bool CachedCompressedReadBuffer::nextImpl()
|
||||
}
|
||||
|
||||
CachedCompressedReadBuffer::CachedCompressedReadBuffer(
|
||||
const std::string & path_, std::function<std::unique_ptr<ReadBufferFromFileBase>()> file_in_creator_, UncompressedCache * cache_)
|
||||
const std::string & path_, std::function<std::unique_ptr<ReadBufferFromFileBase>()> file_in_creator_, UncompressedCache * cache_, bool allow_different_codecs_)
|
||||
: ReadBuffer(nullptr, 0), file_in_creator(std::move(file_in_creator_)), cache(cache_), path(path_), file_pos(0)
|
||||
{
|
||||
allow_different_codecs = allow_different_codecs_;
|
||||
}
|
||||
|
||||
void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block)
|
||||
|
||||
@ -38,7 +38,7 @@ private:
|
||||
clockid_t clock_type {};
|
||||
|
||||
public:
|
||||
CachedCompressedReadBuffer(const std::string & path, std::function<std::unique_ptr<ReadBufferFromFileBase>()> file_in_creator, UncompressedCache * cache_);
|
||||
CachedCompressedReadBuffer(const std::string & path, std::function<std::unique_ptr<ReadBufferFromFileBase>()> file_in_creator, UncompressedCache * cache_, bool allow_different_codecs_ = false);
|
||||
|
||||
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
|
||||
|
||||
|
||||
@ -105,13 +105,24 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
|
||||
uint8_t method = ICompressionCodec::readMethod(own_compressed_buffer.data());
|
||||
|
||||
if (!codec)
|
||||
{
|
||||
codec = CompressionCodecFactory::instance().get(method);
|
||||
}
|
||||
else if (method != codec->getMethodByte())
|
||||
throw Exception("Data compressed with different methods, given method byte 0x"
|
||||
+ getHexUIntLowercase(method)
|
||||
+ ", previous method byte 0x"
|
||||
+ getHexUIntLowercase(codec->getMethodByte()),
|
||||
ErrorCodes::CANNOT_DECOMPRESS);
|
||||
{
|
||||
if (allow_different_codecs)
|
||||
{
|
||||
codec = CompressionCodecFactory::instance().get(method);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception("Data compressed with different methods, given method byte 0x"
|
||||
+ getHexUIntLowercase(method)
|
||||
+ ", previous method byte 0x"
|
||||
+ getHexUIntLowercase(codec->getMethodByte()),
|
||||
ErrorCodes::CANNOT_DECOMPRESS);
|
||||
}
|
||||
}
|
||||
|
||||
size_compressed_without_checksum = ICompressionCodec::readCompressedBlockSize(own_compressed_buffer.data());
|
||||
size_decompressed = ICompressionCodec::readDecompressedBlockSize(own_compressed_buffer.data());
|
||||
@ -163,21 +174,32 @@ void CompressedReadBufferBase::decompress(char * to, size_t size_decompressed, s
|
||||
uint8_t method = ICompressionCodec::readMethod(compressed_buffer);
|
||||
|
||||
if (!codec)
|
||||
{
|
||||
codec = CompressionCodecFactory::instance().get(method);
|
||||
}
|
||||
else if (codec->getMethodByte() != method)
|
||||
throw Exception("Data compressed with different methods, given method byte "
|
||||
+ getHexUIntLowercase(method)
|
||||
+ ", previous method byte "
|
||||
+ getHexUIntLowercase(codec->getMethodByte()),
|
||||
ErrorCodes::CANNOT_DECOMPRESS);
|
||||
{
|
||||
if (allow_different_codecs)
|
||||
{
|
||||
codec = CompressionCodecFactory::instance().get(method);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception("Data compressed with different methods, given method byte "
|
||||
+ getHexUIntLowercase(method)
|
||||
+ ", previous method byte "
|
||||
+ getHexUIntLowercase(codec->getMethodByte()),
|
||||
ErrorCodes::CANNOT_DECOMPRESS);
|
||||
}
|
||||
}
|
||||
|
||||
codec->decompress(compressed_buffer, size_compressed_without_checksum, to);
|
||||
}
|
||||
|
||||
|
||||
/// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'.
|
||||
CompressedReadBufferBase::CompressedReadBufferBase(ReadBuffer * in)
|
||||
: compressed_in(in), own_compressed_buffer(0)
|
||||
CompressedReadBufferBase::CompressedReadBufferBase(ReadBuffer * in, bool allow_different_codecs_)
|
||||
: compressed_in(in), own_compressed_buffer(0), allow_different_codecs(allow_different_codecs_)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,9 @@ protected:
|
||||
/// Don't checksum on decompressing.
|
||||
bool disable_checksum = false;
|
||||
|
||||
/// Allow reading data, compressed by different codecs from one file.
|
||||
bool allow_different_codecs;
|
||||
|
||||
/// Read compressed data into compressed_buffer. Get size of decompressed data from block header. Checksum if need.
|
||||
/// Returns number of compressed bytes read.
|
||||
size_t readCompressedData(size_t & size_decompressed, size_t & size_compressed_without_checksum);
|
||||
@ -34,7 +37,7 @@ protected:
|
||||
|
||||
public:
|
||||
/// 'compressed_in' could be initialized lazily, but before first call of 'readCompressedData'.
|
||||
CompressedReadBufferBase(ReadBuffer * in = nullptr);
|
||||
CompressedReadBufferBase(ReadBuffer * in = nullptr, bool allow_different_codecs_ = false);
|
||||
~CompressedReadBufferBase();
|
||||
|
||||
/** Disable checksums.
|
||||
|
||||
@ -36,20 +36,22 @@ bool CompressedReadBufferFromFile::nextImpl()
|
||||
return true;
|
||||
}
|
||||
|
||||
CompressedReadBufferFromFile::CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf)
|
||||
CompressedReadBufferFromFile::CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_)
|
||||
: BufferWithOwnMemory<ReadBuffer>(0), p_file_in(std::move(buf)), file_in(*p_file_in)
|
||||
{
|
||||
compressed_in = &file_in;
|
||||
allow_different_codecs = allow_different_codecs_;
|
||||
}
|
||||
|
||||
|
||||
CompressedReadBufferFromFile::CompressedReadBufferFromFile(
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, size_t buf_size)
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, size_t buf_size, bool allow_different_codecs_)
|
||||
: BufferWithOwnMemory<ReadBuffer>(0)
|
||||
, p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, buf_size))
|
||||
, file_in(*p_file_in)
|
||||
{
|
||||
compressed_in = &file_in;
|
||||
allow_different_codecs = allow_different_codecs_;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -28,10 +28,11 @@ private:
|
||||
bool nextImpl() override;
|
||||
|
||||
public:
|
||||
CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf);
|
||||
CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
|
||||
|
||||
CompressedReadBufferFromFile(
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold,
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, bool allow_different_codecs_ = false);
|
||||
|
||||
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
|
||||
|
||||
|
||||
@ -36,6 +36,11 @@ ASTPtr CompressionCodecDelta::getCodecDesc() const
|
||||
return makeASTFunction("Delta", literal);
|
||||
}
|
||||
|
||||
void CompressionCodecDelta::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
|
||||
@ -14,7 +14,10 @@ public:
|
||||
|
||||
ASTPtr getCodecDesc() const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -339,6 +339,12 @@ ASTPtr CompressionCodecDoubleDelta::getCodecDesc() const
|
||||
return std::make_shared<ASTIdentifier>("DoubleDelta");
|
||||
}
|
||||
|
||||
void CompressionCodecDoubleDelta::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
hash.update(data_bytes_size);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecDoubleDelta::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
const auto result = 2 // common header
|
||||
|
||||
@ -100,7 +100,10 @@ public:
|
||||
|
||||
ASTPtr getCodecDesc() const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -254,6 +254,12 @@ ASTPtr CompressionCodecGorilla::getCodecDesc() const
|
||||
return std::make_shared<ASTIdentifier>("Gorilla");
|
||||
}
|
||||
|
||||
void CompressionCodecGorilla::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
hash.update(data_bytes_size);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecGorilla::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
const auto result = 2 // common header
|
||||
|
||||
@ -97,7 +97,10 @@ public:
|
||||
|
||||
ASTPtr getCodecDesc() const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -35,6 +35,11 @@ ASTPtr CompressionCodecLZ4::getCodecDesc() const
|
||||
return std::make_shared<ASTIdentifier>("LZ4");
|
||||
}
|
||||
|
||||
void CompressionCodecLZ4::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecLZ4::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
return LZ4_COMPRESSBOUND(uncompressed_size);
|
||||
|
||||
@ -18,6 +18,8 @@ public:
|
||||
|
||||
UInt32 getAdditionalSizeAtTheEndOfBuffer() const override { return LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER; }
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
|
||||
@ -37,6 +37,12 @@ ASTPtr CompressionCodecMultiple::getCodecDesc() const
|
||||
return result;
|
||||
}
|
||||
|
||||
void CompressionCodecMultiple::updateHash(SipHash & hash) const
|
||||
{
|
||||
for (const auto & codec : codecs)
|
||||
codec->updateHash(hash);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
UInt32 compressed_size = uncompressed_size;
|
||||
|
||||
@ -19,7 +19,10 @@ public:
|
||||
|
||||
static std::vector<uint8_t> getCodecsBytesFromData(const char * source);
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const override;
|
||||
|
||||
@ -17,6 +17,11 @@ ASTPtr CompressionCodecNone::getCodecDesc() const
|
||||
return std::make_shared<ASTIdentifier>("NONE");
|
||||
}
|
||||
|
||||
void CompressionCodecNone::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecNone::doCompressData(const char * source, UInt32 source_size, char * dest) const
|
||||
{
|
||||
memcpy(dest, source, source_size);
|
||||
|
||||
@ -15,7 +15,10 @@ public:
|
||||
|
||||
ASTPtr getCodecDesc() const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -646,6 +646,13 @@ ASTPtr CompressionCodecT64::getCodecDesc() const
|
||||
return makeASTFunction("T64", literal);
|
||||
}
|
||||
|
||||
void CompressionCodecT64::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
hash.update(type_idx);
|
||||
hash.update(variant);
|
||||
}
|
||||
|
||||
void registerCodecT64(CompressionCodecFactory & factory)
|
||||
{
|
||||
auto reg_func = [&](const ASTPtr & arguments, DataTypePtr type) -> CompressionCodecPtr
|
||||
|
||||
@ -35,6 +35,8 @@ public:
|
||||
|
||||
ASTPtr getCodecDesc() const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
UInt32 doCompressData(const char * src, UInt32 src_size, char * dst) const override;
|
||||
void doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -32,6 +32,11 @@ ASTPtr CompressionCodecZSTD::getCodecDesc() const
|
||||
return makeASTFunction("ZSTD", literal);
|
||||
}
|
||||
|
||||
void CompressionCodecZSTD::updateHash(SipHash & hash) const
|
||||
{
|
||||
getCodecDesc()->updateTreeHash(hash);
|
||||
}
|
||||
|
||||
UInt32 CompressionCodecZSTD::getMaxCompressedDataSize(UInt32 uncompressed_size) const
|
||||
{
|
||||
return ZSTD_compressBound(uncompressed_size);
|
||||
|
||||
@ -21,7 +21,10 @@ public:
|
||||
|
||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
||||
|
||||
void updateHash(SipHash & hash) const override;
|
||||
|
||||
protected:
|
||||
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||
|
||||
@ -35,6 +35,13 @@ ASTPtr ICompressionCodec::getFullCodecDesc() const
|
||||
return result;
|
||||
}
|
||||
|
||||
UInt64 ICompressionCodec::getHash() const
|
||||
{
|
||||
SipHash hash;
|
||||
updateHash(hash);
|
||||
return hash.get64();
|
||||
}
|
||||
|
||||
UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char * dest) const
|
||||
{
|
||||
assert(source != nullptr && dest != nullptr);
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Compression/CompressionInfo.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -36,6 +37,10 @@ public:
|
||||
/// "CODEC(LZ4,LZ4HC(5))"
|
||||
ASTPtr getFullCodecDesc() const;
|
||||
|
||||
/// Hash, that depends on codec ast and optional parameters like data type
|
||||
virtual void updateHash(SipHash & hash) const = 0;
|
||||
UInt64 getHash() const;
|
||||
|
||||
/// Compressed bytes from uncompressed source to dest. Dest should preallocate memory
|
||||
UInt32 compress(const char * source, UInt32 source_size, char * dest) const;
|
||||
|
||||
|
||||
@ -129,7 +129,7 @@ private:
|
||||
Shift shift;
|
||||
if (decimal0 && decimal1)
|
||||
{
|
||||
auto result_type = decimalResultType(*decimal0, *decimal1, false, false);
|
||||
auto result_type = decimalResultType<false, false>(*decimal0, *decimal1);
|
||||
shift.a = static_cast<CompareInt>(result_type.scaleFactorFor(*decimal0, false).value);
|
||||
shift.b = static_cast<CompareInt>(result_type.scaleFactorFor(*decimal1, false).value);
|
||||
}
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
#include "ExternalResultDescription.h"
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
@ -64,6 +66,14 @@ void ExternalResultDescription::init(const Block & sample_block_)
|
||||
types.emplace_back(ValueType::vtString, is_nullable);
|
||||
else if (typeid_cast<const DataTypeEnum16 *>(type))
|
||||
types.emplace_back(ValueType::vtString, is_nullable);
|
||||
else if (typeid_cast<const DataTypeDateTime64 *>(type))
|
||||
types.emplace_back(ValueType::vtDateTime64, is_nullable);
|
||||
else if (typeid_cast<const DataTypeDecimal<Decimal32> *>(type))
|
||||
types.emplace_back(ValueType::vtDecimal32, is_nullable);
|
||||
else if (typeid_cast<const DataTypeDecimal<Decimal64> *>(type))
|
||||
types.emplace_back(ValueType::vtDecimal64, is_nullable);
|
||||
else if (typeid_cast<const DataTypeDecimal<Decimal128> *>(type))
|
||||
types.emplace_back(ValueType::vtDecimal128, is_nullable);
|
||||
else
|
||||
throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE};
|
||||
}
|
||||
|
||||
@ -26,6 +26,10 @@ struct ExternalResultDescription
|
||||
vtDate,
|
||||
vtDateTime,
|
||||
vtUUID,
|
||||
vtDateTime64,
|
||||
vtDecimal32,
|
||||
vtDecimal64,
|
||||
vtDecimal128
|
||||
};
|
||||
|
||||
Block sample_block;
|
||||
|
||||
99
src/Core/MultiEnum.h
Normal file
99
src/Core/MultiEnum.h
Normal file
@ -0,0 +1,99 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
||||
// Wrapper around enum that can have multiple values (or none) set at once.
|
||||
template <typename EnumTypeT, typename StorageTypeT = std::uint64_t>
|
||||
struct MultiEnum
|
||||
{
|
||||
using StorageType = StorageTypeT;
|
||||
using EnumType = EnumTypeT;
|
||||
|
||||
MultiEnum() = default;
|
||||
|
||||
template <typename ... EnumValues, typename = std::enable_if_t<std::conjunction_v<std::is_same<EnumTypeT, EnumValues>...>>>
|
||||
explicit MultiEnum(EnumValues ... v)
|
||||
: MultiEnum((toBitFlag(v) | ... | 0u))
|
||||
{}
|
||||
|
||||
template <typename ValueType, typename = std::enable_if_t<std::is_convertible_v<ValueType, StorageType>>>
|
||||
explicit MultiEnum(ValueType v)
|
||||
: bitset(v)
|
||||
{
|
||||
static_assert(std::is_unsigned_v<ValueType>);
|
||||
static_assert(std::is_unsigned_v<StorageType> && std::is_integral_v<StorageType>);
|
||||
}
|
||||
|
||||
MultiEnum(const MultiEnum & other) = default;
|
||||
MultiEnum & operator=(const MultiEnum & other) = default;
|
||||
|
||||
bool isSet(EnumType value) const
|
||||
{
|
||||
return bitset & toBitFlag(value);
|
||||
}
|
||||
|
||||
void set(EnumType value)
|
||||
{
|
||||
bitset |= toBitFlag(value);
|
||||
}
|
||||
|
||||
void unSet(EnumType value)
|
||||
{
|
||||
bitset &= ~(toBitFlag(value));
|
||||
}
|
||||
|
||||
void reset()
|
||||
{
|
||||
bitset = 0;
|
||||
}
|
||||
|
||||
StorageType getValue() const
|
||||
{
|
||||
return bitset;
|
||||
}
|
||||
|
||||
template <typename ValueType, typename = std::enable_if_t<std::is_convertible_v<ValueType, StorageType>>>
|
||||
void setValue(ValueType new_value)
|
||||
{
|
||||
// Can't set value from any enum avoid confusion
|
||||
static_assert(!std::is_enum_v<ValueType>);
|
||||
bitset = new_value;
|
||||
}
|
||||
|
||||
bool operator==(const MultiEnum & other) const
|
||||
{
|
||||
return bitset == other.bitset;
|
||||
}
|
||||
|
||||
template <typename ValueType, typename = std::enable_if_t<std::is_convertible_v<ValueType, StorageType>>>
|
||||
bool operator==(ValueType other) const
|
||||
{
|
||||
// Shouldn't be comparable with any enum to avoid confusion
|
||||
static_assert(!std::is_enum_v<ValueType>);
|
||||
return bitset == other;
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
bool operator!=(U && other) const
|
||||
{
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
template <typename ValueType, typename = std::enable_if_t<std::is_convertible_v<ValueType, StorageType>>>
|
||||
friend bool operator==(ValueType left, MultiEnum right)
|
||||
{
|
||||
return right == left;
|
||||
}
|
||||
|
||||
template <typename L>
|
||||
friend bool operator!=(L left, MultiEnum right)
|
||||
{
|
||||
return !(right == left);
|
||||
}
|
||||
|
||||
private:
|
||||
StorageType bitset = 0;
|
||||
|
||||
static StorageType toBitFlag(EnumType v) { return StorageType{1} << static_cast<StorageType>(v); }
|
||||
};
|
||||
@ -383,6 +383,7 @@ class IColumn;
|
||||
M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \
|
||||
M(Bool, allow_experimental_database_materialize_mysql, false, "Allow to create database with Engine=MaterializeMySQL(...).", 0) \
|
||||
M(Bool, system_events_show_zero_values, false, "Include all metrics, even with zero values", 0) \
|
||||
M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \
|
||||
\
|
||||
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
|
||||
\
|
||||
@ -440,6 +441,7 @@ class IColumn;
|
||||
M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \
|
||||
M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \
|
||||
M(Bool, output_format_tsv_crlf_end_of_line, false, "If it is set true, end of line in TSV format will be \\r\\n instead of \\n.", 0) \
|
||||
M(String, output_format_tsv_null_representation, "\\N", "Custom NULL representation in TSV format", 0) \
|
||||
\
|
||||
M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \
|
||||
M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \
|
||||
|
||||
@ -11,6 +11,7 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_DISTRIBUTED_PRODUCT_MODE;
|
||||
extern const int UNKNOWN_JOIN;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL;
|
||||
}
|
||||
|
||||
|
||||
@ -91,4 +92,8 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUME
|
||||
{{"Ordinary", DefaultDatabaseEngine::Ordinary},
|
||||
{"Atomic", DefaultDatabaseEngine::Atomic}})
|
||||
|
||||
IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL,
|
||||
{{"decimal", MySQLDataTypesSupport::DECIMAL},
|
||||
{"datetime64", MySQLDataTypesSupport::DATETIME64}})
|
||||
|
||||
}
|
||||
|
||||
@ -126,4 +126,15 @@ enum class DefaultDatabaseEngine
|
||||
};
|
||||
|
||||
DECLARE_SETTING_ENUM(DefaultDatabaseEngine)
|
||||
|
||||
|
||||
enum class MySQLDataTypesSupport
|
||||
{
|
||||
DECIMAL, // convert MySQL's decimal and number to ClickHouse Decimal when applicable
|
||||
DATETIME64, // convert MySQL's DATETIME and TIMESTAMP and ClickHouse DateTime64 if precision is > 0 or range is greater that for DateTime.
|
||||
// ENUM
|
||||
};
|
||||
|
||||
DECLARE_SETTING_MULTI_ENUM(MySQLDataTypesSupport)
|
||||
|
||||
}
|
||||
|
||||
@ -4,9 +4,11 @@
|
||||
#include <Poco/URI.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/Field.h>
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
#include <chrono>
|
||||
#include <unordered_map>
|
||||
#include <string_view>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -328,6 +330,113 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
throw Exception(msg, ERROR_CODE_FOR_UNEXPECTED_NAME); \
|
||||
}
|
||||
|
||||
// Mostly like SettingFieldEnum, but can have multiple enum values (or none) set at once.
|
||||
template <typename Enum, typename Traits>
|
||||
struct SettingFieldMultiEnum
|
||||
{
|
||||
using EnumType = Enum;
|
||||
using ValueType = MultiEnum<Enum>;
|
||||
using StorageType = typename ValueType::StorageType;
|
||||
|
||||
ValueType value;
|
||||
bool changed = false;
|
||||
|
||||
explicit SettingFieldMultiEnum(ValueType v = ValueType{}) : value{v} {}
|
||||
explicit SettingFieldMultiEnum(EnumType e) : value{e} {}
|
||||
explicit SettingFieldMultiEnum(StorageType s) : value(s) {}
|
||||
explicit SettingFieldMultiEnum(const Field & f) : value(parseValueFromString(f.safeGet<const String &>())) {}
|
||||
|
||||
operator ValueType() const { return value; }
|
||||
explicit operator StorageType() const { return value.getValue(); }
|
||||
explicit operator Field() const { return toString(); }
|
||||
|
||||
SettingFieldMultiEnum & operator= (StorageType x) { changed = x != value.getValue(); value.setValue(x); return *this; }
|
||||
SettingFieldMultiEnum & operator= (ValueType x) { changed = !(x == value); value = x; return *this; }
|
||||
SettingFieldMultiEnum & operator= (const Field & x) { parseFromString(x.safeGet<const String &>()); return *this; }
|
||||
|
||||
String toString() const
|
||||
{
|
||||
static const String separator = ",";
|
||||
String result;
|
||||
for (StorageType i = 0; i < Traits::getEnumSize(); ++i)
|
||||
{
|
||||
const auto v = static_cast<Enum>(i);
|
||||
if (value.isSet(v))
|
||||
{
|
||||
result += Traits::toString(v);
|
||||
result += separator;
|
||||
}
|
||||
}
|
||||
|
||||
if (result.size() > 0)
|
||||
result.erase(result.size() - separator.size());
|
||||
|
||||
return result;
|
||||
}
|
||||
void parseFromString(const String & str) { *this = parseValueFromString(str); }
|
||||
|
||||
void writeBinary(WriteBuffer & out) const;
|
||||
void readBinary(ReadBuffer & in);
|
||||
|
||||
private:
|
||||
static ValueType parseValueFromString(const std::string_view str)
|
||||
{
|
||||
static const String separators=", ";
|
||||
|
||||
ValueType result;
|
||||
|
||||
//to avoid allocating memory on substr()
|
||||
const std::string_view str_view{str};
|
||||
|
||||
auto value_start = str_view.find_first_not_of(separators);
|
||||
while (value_start != std::string::npos)
|
||||
{
|
||||
auto value_end = str_view.find_first_of(separators, value_start + 1);
|
||||
if (value_end == std::string::npos)
|
||||
value_end = str_view.size();
|
||||
|
||||
result.set(Traits::fromString(str_view.substr(value_start, value_end - value_start)));
|
||||
value_start = str_view.find_first_not_of(separators, value_end);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename EnumT, typename Traits>
|
||||
void SettingFieldMultiEnum<EnumT, Traits>::writeBinary(WriteBuffer & out) const
|
||||
{
|
||||
SettingFieldEnumHelpers::writeBinary(toString(), out);
|
||||
}
|
||||
|
||||
template <typename EnumT, typename Traits>
|
||||
void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
{
|
||||
parseFromString(SettingFieldEnumHelpers::readBinary(in));
|
||||
}
|
||||
|
||||
#define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \
|
||||
DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
|
||||
|
||||
#define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \
|
||||
struct SettingField##NEW_NAME##Traits \
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
static size_t getEnumSize(); \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(const std::string_view & str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
|
||||
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)
|
||||
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\
|
||||
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
|
||||
return std::initializer_list<std::pair<const char*, NEW_NAME>> __VA_ARGS__ .size();\
|
||||
}
|
||||
|
||||
/// Can keep a value of any type. Used for user-defined settings.
|
||||
struct SettingFieldCustom
|
||||
|
||||
158
src/Core/tests/gtest_multienum.cpp
Normal file
158
src/Core/tests/gtest_multienum.cpp
Normal file
@ -0,0 +1,158 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <type_traits>
|
||||
#include <Core/MultiEnum.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
using namespace DB;
|
||||
enum class TestEnum : UInt8
|
||||
{
|
||||
// name represents which bit is going to be set
|
||||
ZERO,
|
||||
ONE,
|
||||
TWO,
|
||||
THREE,
|
||||
FOUR,
|
||||
FIVE
|
||||
};
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, WithDefault)
|
||||
{
|
||||
MultiEnum<TestEnum, UInt8> multi_enum;
|
||||
ASSERT_EQ(0, multi_enum.getValue());
|
||||
ASSERT_EQ(0, multi_enum);
|
||||
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ZERO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ONE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::TWO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::THREE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FOUR));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FIVE));
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, WitheEnum)
|
||||
{
|
||||
MultiEnum<TestEnum, UInt8> multi_enum(TestEnum::FOUR);
|
||||
ASSERT_EQ(16, multi_enum.getValue());
|
||||
ASSERT_EQ(16, multi_enum);
|
||||
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ZERO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ONE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::TWO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::THREE));
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::FOUR));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FIVE));
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, WithValue)
|
||||
{
|
||||
const MultiEnum<TestEnum> multi_enum(13u); // (1 | (1 << 2 | 1 << 3)
|
||||
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::ZERO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ONE));
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::TWO));
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::THREE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FOUR));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FIVE));
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, WithMany)
|
||||
{
|
||||
MultiEnum<TestEnum> multi_enum{TestEnum::ONE, TestEnum::FIVE};
|
||||
ASSERT_EQ(1 << 1 | 1 << 5, multi_enum.getValue());
|
||||
ASSERT_EQ(1 << 1 | 1 << 5, multi_enum);
|
||||
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::ZERO));
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::ONE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::TWO));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::THREE));
|
||||
ASSERT_FALSE(multi_enum.isSet(TestEnum::FOUR));
|
||||
ASSERT_TRUE(multi_enum.isSet(TestEnum::FIVE));
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, WithCopyConstructor)
|
||||
{
|
||||
const MultiEnum<TestEnum> multi_enum_source{TestEnum::ONE, TestEnum::FIVE};
|
||||
MultiEnum<TestEnum> multi_enum{multi_enum_source};
|
||||
|
||||
ASSERT_EQ(1 << 1 | 1 << 5, multi_enum.getValue());
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, SetAndUnSet)
|
||||
{
|
||||
MultiEnum<TestEnum> multi_enum;
|
||||
multi_enum.set(TestEnum::ONE);
|
||||
ASSERT_EQ(1 << 1, multi_enum);
|
||||
|
||||
multi_enum.set(TestEnum::TWO);
|
||||
ASSERT_EQ(1 << 1| (1 << 2), multi_enum);
|
||||
|
||||
multi_enum.unSet(TestEnum::ONE);
|
||||
ASSERT_EQ(1 << 2, multi_enum);
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, SetValueOnDifferentTypes)
|
||||
{
|
||||
MultiEnum<TestEnum> multi_enum;
|
||||
|
||||
multi_enum.setValue(static_cast<UInt8>(1));
|
||||
ASSERT_EQ(1, multi_enum);
|
||||
|
||||
multi_enum.setValue(static_cast<UInt16>(2));
|
||||
ASSERT_EQ(2, multi_enum);
|
||||
|
||||
multi_enum.setValue(static_cast<UInt32>(3));
|
||||
ASSERT_EQ(3, multi_enum);
|
||||
|
||||
multi_enum.setValue(static_cast<UInt64>(4));
|
||||
ASSERT_EQ(4, multi_enum);
|
||||
}
|
||||
|
||||
// shouldn't compile
|
||||
//GTEST_TEST(MultiEnum, WithOtherEnumType)
|
||||
//{
|
||||
// MultiEnum<TestEnum> multi_enum;
|
||||
|
||||
// enum FOO {BAR, FOOBAR};
|
||||
// MultiEnum<TestEnum> multi_enum2(BAR);
|
||||
// MultiEnum<TestEnum> multi_enum3(BAR, FOOBAR);
|
||||
// multi_enum.setValue(FOO::BAR);
|
||||
// multi_enum == FOO::BAR;
|
||||
// FOO::BAR == multi_enum;
|
||||
//}
|
||||
|
||||
GTEST_TEST(MultiEnum, SetSameValueMultipleTimes)
|
||||
{
|
||||
// Setting same value is idempotent.
|
||||
MultiEnum<TestEnum> multi_enum;
|
||||
multi_enum.set(TestEnum::ONE);
|
||||
ASSERT_EQ(1 << 1, multi_enum);
|
||||
|
||||
multi_enum.set(TestEnum::ONE);
|
||||
ASSERT_EQ(1 << 1, multi_enum);
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, UnSetValuesThatWerentSet)
|
||||
{
|
||||
// Unsetting values that weren't set shouldn't change other flags nor aggregate value.
|
||||
MultiEnum<TestEnum> multi_enum{TestEnum::ONE, TestEnum::THREE};
|
||||
multi_enum.unSet(TestEnum::TWO);
|
||||
ASSERT_EQ(1 << 1 | 1 << 3, multi_enum);
|
||||
|
||||
multi_enum.unSet(TestEnum::FOUR);
|
||||
ASSERT_EQ(1 << 1 | 1 << 3, multi_enum);
|
||||
|
||||
multi_enum.unSet(TestEnum::FIVE);
|
||||
ASSERT_EQ(1 << 1 | 1 << 3, multi_enum);
|
||||
}
|
||||
|
||||
GTEST_TEST(MultiEnum, Reset)
|
||||
{
|
||||
MultiEnum<TestEnum> multi_enum{TestEnum::ONE, TestEnum::THREE};
|
||||
multi_enum.reset();
|
||||
ASSERT_EQ(0, multi_enum);
|
||||
}
|
||||
146
src/Core/tests/gtest_settings.cpp
Normal file
146
src/Core/tests/gtest_settings.cpp
Normal file
@ -0,0 +1,146 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Core/SettingsFields.h>
|
||||
#include <Core/SettingsEnums.h>
|
||||
#include <Core/Field.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
using namespace DB;
|
||||
using SettingMySQLDataTypesSupport = SettingFieldMultiEnum<MySQLDataTypesSupport, SettingFieldMySQLDataTypesSupportTraits>;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename Enum, typename Traits>
|
||||
bool operator== (const SettingFieldMultiEnum<Enum, Traits> & setting, const Field & f)
|
||||
{
|
||||
return Field(setting) == f;
|
||||
}
|
||||
|
||||
template <typename Enum, typename Traits>
|
||||
bool operator== (const Field & f, const SettingFieldMultiEnum<Enum, Traits> & setting)
|
||||
{
|
||||
return f == Field(setting);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
GTEST_TEST(MySQLDataTypesSupport, WithDefault)
|
||||
{
|
||||
// Setting can be default-initialized and that means all values are unset.
|
||||
const SettingMySQLDataTypesSupport setting;
|
||||
ASSERT_EQ(0, setting.value.getValue());
|
||||
ASSERT_EQ("", setting.toString());
|
||||
ASSERT_EQ(setting, Field(""));
|
||||
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
}
|
||||
|
||||
GTEST_TEST(SettingMySQLDataTypesSupport, WithDECIMAL)
|
||||
{
|
||||
// Setting can be initialized with MySQLDataTypesSupport::DECIMAL
|
||||
// and this value can be obtained in varios forms with getters.
|
||||
const SettingMySQLDataTypesSupport setting(MySQLDataTypesSupport::DECIMAL);
|
||||
ASSERT_EQ(1, setting.value.getValue());
|
||||
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
|
||||
ASSERT_EQ("decimal", setting.toString());
|
||||
ASSERT_EQ(Field("decimal"), setting);
|
||||
}
|
||||
|
||||
GTEST_TEST(SettingMySQLDataTypesSupport, With1)
|
||||
{
|
||||
// Setting can be initialized with int value corresponding to DECIMAL
|
||||
// and rest of the test is the same as for that value.
|
||||
const SettingMySQLDataTypesSupport setting(1u);
|
||||
ASSERT_EQ(1, setting.value.getValue());
|
||||
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
|
||||
ASSERT_EQ("decimal", setting.toString());
|
||||
ASSERT_EQ(Field("decimal"), setting);
|
||||
}
|
||||
|
||||
GTEST_TEST(SettingMySQLDataTypesSupport, WithMultipleValues)
|
||||
{
|
||||
// Setting can be initialized with int value corresponding to (DECIMAL | DATETIME64)
|
||||
const SettingMySQLDataTypesSupport setting(3u);
|
||||
ASSERT_EQ(3, setting.value.getValue());
|
||||
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
|
||||
ASSERT_EQ("decimal,datetime64", setting.toString());
|
||||
ASSERT_EQ(Field("decimal,datetime64"), setting);
|
||||
}
|
||||
|
||||
GTEST_TEST(SettingMySQLDataTypesSupport, SetString)
|
||||
{
|
||||
SettingMySQLDataTypesSupport setting;
|
||||
setting = String("decimal");
|
||||
ASSERT_TRUE(setting.changed);
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("decimal", setting.toString());
|
||||
ASSERT_EQ(Field("decimal"), setting);
|
||||
|
||||
setting = "datetime64,decimal";
|
||||
ASSERT_TRUE(setting.changed);
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("decimal,datetime64", setting.toString());
|
||||
ASSERT_EQ(Field("decimal,datetime64"), setting);
|
||||
|
||||
// comma with spaces
|
||||
setting = " datetime64 , decimal ";
|
||||
ASSERT_FALSE(setting.changed); // false since value is the same as previous one.
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("decimal,datetime64", setting.toString());
|
||||
ASSERT_EQ(Field("decimal,datetime64"), setting);
|
||||
|
||||
setting = String(",,,,,,,, ,decimal");
|
||||
ASSERT_TRUE(setting.changed);
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("decimal", setting.toString());
|
||||
ASSERT_EQ(Field("decimal"), setting);
|
||||
|
||||
setting = String(",decimal,decimal,decimal,decimal,decimal,decimal,decimal,decimal,decimal,");
|
||||
ASSERT_FALSE(setting.changed); //since previous value was DECIMAL
|
||||
ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("decimal", setting.toString());
|
||||
ASSERT_EQ(Field("decimal"), setting);
|
||||
|
||||
setting = String("");
|
||||
ASSERT_TRUE(setting.changed);
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL));
|
||||
ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64));
|
||||
ASSERT_EQ("", setting.toString());
|
||||
ASSERT_EQ(Field(""), setting);
|
||||
}
|
||||
|
||||
GTEST_TEST(SettingMySQLDataTypesSupport, SetInvalidString)
|
||||
{
|
||||
// Setting can be initialized with int value corresponding to (DECIMAL | DATETIME64)
|
||||
SettingMySQLDataTypesSupport setting;
|
||||
EXPECT_THROW(setting = String("FOOBAR"), Exception);
|
||||
ASSERT_FALSE(setting.changed);
|
||||
ASSERT_EQ(0, setting.value.getValue());
|
||||
|
||||
EXPECT_THROW(setting = String("decimal,datetime64,123"), Exception);
|
||||
ASSERT_FALSE(setting.changed);
|
||||
ASSERT_EQ(0, setting.value.getValue());
|
||||
|
||||
EXPECT_NO_THROW(setting = String(", "));
|
||||
ASSERT_FALSE(setting.changed);
|
||||
ASSERT_EQ(0, setting.value.getValue());
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ String ExpressionBlockInputStream::getName() const { return "Expression"; }
|
||||
Block ExpressionBlockInputStream::getTotals()
|
||||
{
|
||||
totals = children.back()->getTotals();
|
||||
expression->executeOnTotals(totals);
|
||||
expression->execute(totals);
|
||||
|
||||
return totals;
|
||||
}
|
||||
@ -30,14 +30,6 @@ Block ExpressionBlockInputStream::getHeader() const
|
||||
|
||||
Block ExpressionBlockInputStream::readImpl()
|
||||
{
|
||||
if (!initialized)
|
||||
{
|
||||
if (expression->resultIsAlwaysEmpty())
|
||||
return {};
|
||||
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
Block res = children.back()->read();
|
||||
if (res)
|
||||
expression->execute(res);
|
||||
|
||||
@ -25,7 +25,6 @@ public:
|
||||
Block getHeader() const override;
|
||||
|
||||
protected:
|
||||
bool initialized = false;
|
||||
ExpressionActionsPtr expression;
|
||||
|
||||
Block readImpl() override;
|
||||
|
||||
@ -54,7 +54,7 @@ String FilterBlockInputStream::getName() const { return "Filter"; }
|
||||
Block FilterBlockInputStream::getTotals()
|
||||
{
|
||||
totals = children.back()->getTotals();
|
||||
expression->executeOnTotals(totals);
|
||||
expression->execute(totals);
|
||||
|
||||
return totals;
|
||||
}
|
||||
|
||||
@ -37,6 +37,7 @@ namespace ErrorCodes
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int MONGODB_CANNOT_AUTHENTICATE;
|
||||
extern const int NOT_FOUND_COLUMN_IN_BLOCK;
|
||||
extern const int UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
|
||||
@ -298,6 +299,8 @@ namespace
|
||||
ErrorCodes::TYPE_MISMATCH};
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw Exception("Value of unsupported type:" + column.getName(), ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -156,38 +156,31 @@ protected:
|
||||
};
|
||||
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
typename std::enable_if_t<(sizeof(T) >= sizeof(U)), DecimalType<T>>
|
||||
inline decimalResultType(const DecimalType<T> & tx, const DecimalType<U> & ty, bool is_multiply, bool is_divide)
|
||||
template <bool is_multiply, bool is_division, typename T, typename U, template <typename> typename DecimalType>
|
||||
inline auto decimalResultType(const DecimalType<T> & tx, const DecimalType<U> & ty)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
UInt32 scale{};
|
||||
if constexpr (is_multiply)
|
||||
scale = tx.getScale() + ty.getScale();
|
||||
else if (is_divide)
|
||||
else if constexpr (is_division)
|
||||
scale = tx.getScale();
|
||||
return DecimalType<T>(DecimalUtils::maxPrecision<T>(), scale);
|
||||
else
|
||||
scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
|
||||
if constexpr (sizeof(T) < sizeof(U))
|
||||
return DecimalType<U>(DecimalUtils::maxPrecision<U>(), scale);
|
||||
else
|
||||
return DecimalType<T>(DecimalUtils::maxPrecision<T>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DecimalType<U>>
|
||||
inline decimalResultType(const DecimalType<T> & tx, const DecimalType<U> & ty, bool is_multiply, bool is_divide)
|
||||
{
|
||||
UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale());
|
||||
if (is_multiply)
|
||||
scale = tx.getScale() * ty.getScale();
|
||||
else if (is_divide)
|
||||
scale = tx.getScale();
|
||||
return DecimalType<U>(DecimalUtils::maxPrecision<U>(), scale);
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
inline const DecimalType<T> decimalResultType(const DecimalType<T> & tx, const DataTypeNumber<U> &, bool, bool)
|
||||
template <bool, bool, typename T, typename U, template <typename> typename DecimalType>
|
||||
inline const DecimalType<T> decimalResultType(const DecimalType<T> & tx, const DataTypeNumber<U> &)
|
||||
{
|
||||
return DecimalType<T>(DecimalUtils::maxPrecision<T>(), tx.getScale());
|
||||
}
|
||||
|
||||
template <typename T, typename U, template <typename> typename DecimalType>
|
||||
inline const DecimalType<U> decimalResultType(const DataTypeNumber<T> &, const DecimalType<U> & ty, bool, bool)
|
||||
template <bool, bool, typename T, typename U, template <typename> typename DecimalType>
|
||||
inline const DecimalType<U> decimalResultType(const DataTypeNumber<T> &, const DecimalType<U> & ty)
|
||||
{
|
||||
return DecimalType<U>(DecimalUtils::maxPrecision<U>(), ty.getScale());
|
||||
}
|
||||
|
||||
@ -217,7 +217,7 @@ void DataTypeNullable::serializeTextEscaped(const IColumn & column, size_t row_n
|
||||
const ColumnNullable & col = assert_cast<const ColumnNullable &>(column);
|
||||
|
||||
if (col.isNullAt(row_num))
|
||||
writeCString("\\N", ostr);
|
||||
writeString(settings.tsv.null_representation, ostr);
|
||||
else
|
||||
nested_data_type->serializeAsTextEscaped(col.getNestedColumn(), row_num, ostr, settings);
|
||||
}
|
||||
|
||||
@ -2,11 +2,16 @@
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <Core/SettingsEnums.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include "DataTypeDate.h"
|
||||
#include "DataTypeDateTime.h"
|
||||
#include "DataTypeDateTime64.h"
|
||||
#include "DataTypeEnum.h"
|
||||
#include "DataTypesDecimal.h"
|
||||
#include "DataTypeFixedString.h"
|
||||
#include "DataTypeNullable.h"
|
||||
#include "DataTypeString.h"
|
||||
@ -25,52 +30,88 @@ ASTPtr dataTypeConvertToQuery(const DataTypePtr & data_type)
|
||||
return makeASTFunction("Nullable", dataTypeConvertToQuery(typeid_cast<const DataTypeNullable *>(data_type.get())->getNestedType()));
|
||||
}
|
||||
|
||||
DataTypePtr convertMySQLDataType(const std::string & mysql_data_type, bool is_nullable, bool is_unsigned, size_t length)
|
||||
DataTypePtr convertMySQLDataType(MultiEnum<MySQLDataTypesSupport> type_support,
|
||||
const std::string & mysql_data_type,
|
||||
bool is_nullable,
|
||||
bool is_unsigned,
|
||||
size_t length,
|
||||
size_t precision,
|
||||
size_t scale)
|
||||
{
|
||||
DataTypePtr res;
|
||||
if (mysql_data_type == "tinyint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
res = std::make_shared<DataTypeUInt8>();
|
||||
else
|
||||
res = std::make_shared<DataTypeInt8>();
|
||||
}
|
||||
else if (mysql_data_type == "smallint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
res = std::make_shared<DataTypeUInt16>();
|
||||
else
|
||||
res = std::make_shared<DataTypeInt16>();
|
||||
}
|
||||
else if (mysql_data_type == "int" || mysql_data_type == "mediumint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
res = std::make_shared<DataTypeUInt32>();
|
||||
else
|
||||
res = std::make_shared<DataTypeInt32>();
|
||||
}
|
||||
else if (mysql_data_type == "bigint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
res = std::make_shared<DataTypeUInt64>();
|
||||
else
|
||||
res = std::make_shared<DataTypeInt64>();
|
||||
}
|
||||
else if (mysql_data_type == "float")
|
||||
res = std::make_shared<DataTypeFloat32>();
|
||||
else if (mysql_data_type == "double")
|
||||
res = std::make_shared<DataTypeFloat64>();
|
||||
else if (mysql_data_type == "date")
|
||||
res = std::make_shared<DataTypeDate>();
|
||||
else if (mysql_data_type == "datetime" || mysql_data_type == "timestamp")
|
||||
res = std::make_shared<DataTypeDateTime>();
|
||||
else if (mysql_data_type == "binary")
|
||||
res = std::make_shared<DataTypeFixedString>(length);
|
||||
else
|
||||
// we expect mysql_data_type to be either "basic_type" or "type_with_params(param1, param2, ...)"
|
||||
auto data_type = std::string_view(mysql_data_type);
|
||||
const auto param_start_pos = data_type.find("(");
|
||||
const auto type_name = data_type.substr(0, param_start_pos);
|
||||
|
||||
DataTypePtr res = [&]() -> DataTypePtr {
|
||||
if (type_name == "tinyint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
else
|
||||
return std::make_shared<DataTypeInt8>();
|
||||
}
|
||||
if (type_name == "smallint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
return std::make_shared<DataTypeUInt16>();
|
||||
else
|
||||
return std::make_shared<DataTypeInt16>();
|
||||
}
|
||||
if (type_name == "int" || type_name == "mediumint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
return std::make_shared<DataTypeUInt32>();
|
||||
else
|
||||
return std::make_shared<DataTypeInt32>();
|
||||
}
|
||||
if (type_name == "bigint")
|
||||
{
|
||||
if (is_unsigned)
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
else
|
||||
return std::make_shared<DataTypeInt64>();
|
||||
}
|
||||
if (type_name == "float")
|
||||
return std::make_shared<DataTypeFloat32>();
|
||||
if (type_name == "double")
|
||||
return std::make_shared<DataTypeFloat64>();
|
||||
if (type_name == "date")
|
||||
return std::make_shared<DataTypeDate>();
|
||||
if (type_name == "binary")
|
||||
return std::make_shared<DataTypeFixedString>(length);
|
||||
if (type_name == "datetime" || type_name == "timestamp")
|
||||
{
|
||||
if (!type_support.isSet(MySQLDataTypesSupport::DATETIME64))
|
||||
return std::make_shared<DataTypeDateTime>();
|
||||
|
||||
if (type_name == "timestamp" && scale == 0)
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime>();
|
||||
}
|
||||
else if (type_name == "datetime" || type_name == "timestamp")
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime64>(scale);
|
||||
}
|
||||
}
|
||||
|
||||
if (type_support.isSet(MySQLDataTypesSupport::DECIMAL) && (type_name == "numeric" || type_name == "decimal"))
|
||||
{
|
||||
if (precision <= DecimalUtils::maxPrecision<Decimal32>())
|
||||
return std::make_shared<DataTypeDecimal<Decimal32>>(precision, scale);
|
||||
else if (precision <= DecimalUtils::maxPrecision<Decimal64>())
|
||||
return std::make_shared<DataTypeDecimal<Decimal64>>(precision, scale);
|
||||
else if (precision <= DecimalUtils::maxPrecision<Decimal128>())
|
||||
return std::make_shared<DataTypeDecimal<Decimal128>>(precision, scale);
|
||||
}
|
||||
|
||||
/// Also String is fallback for all unknown types.
|
||||
res = std::make_shared<DataTypeString>();
|
||||
return std::make_shared<DataTypeString>();
|
||||
}();
|
||||
|
||||
if (is_nullable)
|
||||
res = std::make_shared<DataTypeNullable>(res);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include "IDataType.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
enum class MySQLDataTypesSupport;
|
||||
|
||||
/// Convert data type to query. for example
|
||||
/// DataTypeUInt8 -> ASTIdentifier(UInt8)
|
||||
/// DataTypeNullable(DataTypeUInt8) -> ASTFunction(ASTIdentifier(UInt8))
|
||||
ASTPtr dataTypeConvertToQuery(const DataTypePtr & data_type);
|
||||
|
||||
/// Convert MySQL type to ClickHouse data type.
|
||||
DataTypePtr convertMySQLDataType(const std::string & mysql_data_type, bool is_nullable, bool is_unsigned, size_t length);
|
||||
DataTypePtr convertMySQLDataType(MultiEnum<MySQLDataTypesSupport> type_support, const std::string & mysql_data_type, bool is_nullable, bool is_unsigned, size_t length, size_t precision, size_t scale);
|
||||
|
||||
}
|
||||
|
||||
101
src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp
Normal file
101
src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp
Normal file
@ -0,0 +1,101 @@
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Core/Field.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
#include <DataTypes/getMostSubtype.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
|
||||
#pragma GCC diagnostic ignored "-Wmissing-declarations"
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <Core/iostream_debug_helpers.h>
|
||||
|
||||
namespace std
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
inline std::ostream& operator<<(std::ostream & ostr, const std::vector<T> & v)
|
||||
{
|
||||
ostr << "[";
|
||||
for (const auto & i : v)
|
||||
{
|
||||
ostr << i << ", ";
|
||||
}
|
||||
return ostr << "] (" << v.size() << ") items";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
struct ParseDataTypeTestCase
|
||||
{
|
||||
const char * type_name;
|
||||
std::vector<String> values;
|
||||
FieldVector expected_values;
|
||||
};
|
||||
|
||||
std::ostream & operator<<(std::ostream & ostr, const ParseDataTypeTestCase & test_case)
|
||||
{
|
||||
return ostr << "ParseDataTypeTestCase{\"" << test_case.type_name << "\", " << test_case.values << "}";
|
||||
}
|
||||
|
||||
|
||||
class ParseDataTypeTest : public ::testing::TestWithParam<ParseDataTypeTestCase>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
const auto & p = GetParam();
|
||||
|
||||
data_type = DataTypeFactory::instance().get(p.type_name);
|
||||
}
|
||||
|
||||
DataTypePtr data_type;
|
||||
};
|
||||
|
||||
TEST_P(ParseDataTypeTest, parseStringValue)
|
||||
{
|
||||
const auto & p = GetParam();
|
||||
|
||||
auto col = data_type->createColumn();
|
||||
for (const auto & value : p.values)
|
||||
{
|
||||
ReadBuffer buffer(const_cast<char *>(value.data()), value.size(), 0);
|
||||
data_type->deserializeAsWholeText(*col, buffer, FormatSettings{});
|
||||
}
|
||||
|
||||
ASSERT_EQ(p.expected_values.size(), col->size()) << "Actual items: " << *col;
|
||||
for (size_t i = 0; i < col->size(); ++i)
|
||||
{
|
||||
ASSERT_EQ(p.expected_values[i], (*col)[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(ParseDecimal,
|
||||
ParseDataTypeTest,
|
||||
::testing::ValuesIn(
|
||||
std::initializer_list<ParseDataTypeTestCase>{
|
||||
{
|
||||
"Decimal(8, 0)",
|
||||
{"0", "5", "8", "-5", "-8", "12345678", "-12345678"},
|
||||
|
||||
std::initializer_list<Field>{
|
||||
DecimalField<Decimal32>(0, 0),
|
||||
DecimalField<Decimal32>(5, 0),
|
||||
DecimalField<Decimal32>(8, 0),
|
||||
DecimalField<Decimal32>(-5, 0),
|
||||
DecimalField<Decimal32>(-8, 0),
|
||||
DecimalField<Decimal32>(12345678, 0),
|
||||
DecimalField<Decimal32>(-12345678, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
@ -10,6 +10,7 @@
|
||||
# include <DataTypes/DataTypesNumber.h>
|
||||
# include <DataTypes/convertMySQLDataType.h>
|
||||
# include <Databases/MySQL/DatabaseConnectionMySQL.h>
|
||||
# include <Databases/MySQL/FetchTablesColumnsList.h>
|
||||
# include <Formats/MySQLBlockInputStream.h>
|
||||
# include <IO/Operators.h>
|
||||
# include <Parsers/ASTCreateQuery.h>
|
||||
@ -43,31 +44,14 @@ constexpr static const auto suffix = ".remove_flag";
|
||||
static constexpr const std::chrono::seconds cleaner_sleep_time{30};
|
||||
static const std::chrono::seconds lock_acquire_timeout{10};
|
||||
|
||||
static String toQueryStringWithQuote(const std::vector<String> & quote_list)
|
||||
{
|
||||
WriteBufferFromOwnString quote_list_query;
|
||||
quote_list_query << "(";
|
||||
|
||||
for (size_t index = 0; index < quote_list.size(); ++index)
|
||||
{
|
||||
if (index)
|
||||
quote_list_query << ",";
|
||||
|
||||
quote_list_query << quote << quote_list[index];
|
||||
}
|
||||
|
||||
quote_list_query << ")";
|
||||
return quote_list_query.str();
|
||||
}
|
||||
|
||||
DatabaseConnectionMySQL::DatabaseConnectionMySQL(
|
||||
const Context & global_context_, const String & database_name_, const String & metadata_path_,
|
||||
DatabaseConnectionMySQL::DatabaseConnectionMySQL(const Context & context, const String & database_name_, const String & metadata_path_,
|
||||
const ASTStorage * database_engine_define_, const String & database_name_in_mysql_, mysqlxx::Pool && pool)
|
||||
: IDatabase(database_name_)
|
||||
, global_context(global_context_.getGlobalContext())
|
||||
, global_context(context.getGlobalContext())
|
||||
, metadata_path(metadata_path_)
|
||||
, database_engine_define(database_engine_define_->clone())
|
||||
, database_name_in_mysql(database_name_in_mysql_)
|
||||
, mysql_datatypes_support_level(context.getQueryContext().getSettingsRef().mysql_datatypes_support_level)
|
||||
, mysql_pool(std::move(pool))
|
||||
{
|
||||
empty(); /// test database is works fine.
|
||||
@ -78,7 +62,7 @@ bool DatabaseConnectionMySQL::empty() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
|
||||
fetchTablesIntoLocalCache();
|
||||
fetchTablesIntoLocalCache(global_context);
|
||||
|
||||
if (local_tables_cache.empty())
|
||||
return true;
|
||||
@ -90,12 +74,12 @@ bool DatabaseConnectionMySQL::empty() const
|
||||
return true;
|
||||
}
|
||||
|
||||
DatabaseTablesIteratorPtr DatabaseConnectionMySQL::getTablesIterator(const Context &, const FilterByNameFunction & filter_by_table_name)
|
||||
DatabaseTablesIteratorPtr DatabaseConnectionMySQL::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name)
|
||||
{
|
||||
Tables tables;
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
|
||||
fetchTablesIntoLocalCache();
|
||||
fetchTablesIntoLocalCache(context);
|
||||
|
||||
for (const auto & [table_name, modify_time_and_storage] : local_tables_cache)
|
||||
if (!remove_or_detach_tables.count(table_name) && (!filter_by_table_name || filter_by_table_name(table_name)))
|
||||
@ -109,11 +93,11 @@ bool DatabaseConnectionMySQL::isTableExist(const String & name, const Context &
|
||||
return bool(tryGetTable(name, context));
|
||||
}
|
||||
|
||||
StoragePtr DatabaseConnectionMySQL::tryGetTable(const String & mysql_table_name, const Context &) const
|
||||
StoragePtr DatabaseConnectionMySQL::tryGetTable(const String & mysql_table_name, const Context & context) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
|
||||
fetchTablesIntoLocalCache();
|
||||
fetchTablesIntoLocalCache(context);
|
||||
|
||||
if (!remove_or_detach_tables.count(mysql_table_name) && local_tables_cache.find(mysql_table_name) != local_tables_cache.end())
|
||||
return local_tables_cache[mysql_table_name].second;
|
||||
@ -157,11 +141,11 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
|
||||
return create_table_query;
|
||||
}
|
||||
|
||||
ASTPtr DatabaseConnectionMySQL::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
|
||||
ASTPtr DatabaseConnectionMySQL::getCreateTableQueryImpl(const String & table_name, const Context & context, bool throw_on_error) const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
|
||||
fetchTablesIntoLocalCache();
|
||||
fetchTablesIntoLocalCache(context);
|
||||
|
||||
if (local_tables_cache.find(table_name) == local_tables_cache.end())
|
||||
{
|
||||
@ -178,7 +162,7 @@ time_t DatabaseConnectionMySQL::getObjectMetadataModificationTime(const String &
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
|
||||
fetchTablesIntoLocalCache();
|
||||
fetchTablesIntoLocalCache(global_context);
|
||||
|
||||
if (local_tables_cache.find(table_name) == local_tables_cache.end())
|
||||
throw Exception("MySQL table " + database_name_in_mysql + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
|
||||
@ -194,12 +178,12 @@ ASTPtr DatabaseConnectionMySQL::getCreateDatabaseQuery() const
|
||||
return create_query;
|
||||
}
|
||||
|
||||
void DatabaseConnectionMySQL::fetchTablesIntoLocalCache() const
|
||||
void DatabaseConnectionMySQL::fetchTablesIntoLocalCache(const Context & context) const
|
||||
{
|
||||
const auto & tables_with_modification_time = fetchTablesWithModificationTime();
|
||||
|
||||
destroyLocalCacheExtraTables(tables_with_modification_time);
|
||||
fetchLatestTablesStructureIntoCache(tables_with_modification_time);
|
||||
fetchLatestTablesStructureIntoCache(tables_with_modification_time, context);
|
||||
}
|
||||
|
||||
void DatabaseConnectionMySQL::destroyLocalCacheExtraTables(const std::map<String, UInt64> & tables_with_modification_time) const
|
||||
@ -216,7 +200,7 @@ void DatabaseConnectionMySQL::destroyLocalCacheExtraTables(const std::map<String
|
||||
}
|
||||
}
|
||||
|
||||
void DatabaseConnectionMySQL::fetchLatestTablesStructureIntoCache(const std::map<String, UInt64> &tables_modification_time) const
|
||||
void DatabaseConnectionMySQL::fetchLatestTablesStructureIntoCache(const std::map<String, UInt64> &tables_modification_time, const Context & context) const
|
||||
{
|
||||
std::vector<String> wait_update_tables_name;
|
||||
for (const auto & table_modification_time : tables_modification_time)
|
||||
@ -228,7 +212,7 @@ void DatabaseConnectionMySQL::fetchLatestTablesStructureIntoCache(const std::map
|
||||
wait_update_tables_name.emplace_back(table_modification_time.first);
|
||||
}
|
||||
|
||||
std::map<String, NamesAndTypesList> tables_and_columns = fetchTablesColumnsList(wait_update_tables_name);
|
||||
std::map<String, NamesAndTypesList> tables_and_columns = fetchTablesColumnsList(wait_update_tables_name, context);
|
||||
|
||||
for (const auto & table_and_columns : tables_and_columns)
|
||||
{
|
||||
@ -280,53 +264,16 @@ std::map<String, UInt64> DatabaseConnectionMySQL::fetchTablesWithModificationTim
|
||||
return tables_with_modification_time;
|
||||
}
|
||||
|
||||
std::map<String, NamesAndTypesList> DatabaseConnectionMySQL::fetchTablesColumnsList(const std::vector<String> & tables_name) const
|
||||
std::map<String, NamesAndTypesList> DatabaseConnectionMySQL::fetchTablesColumnsList(const std::vector<String> & tables_name, const Context & context) const
|
||||
{
|
||||
std::map<String, NamesAndTypesList> tables_and_columns;
|
||||
const auto & settings = context.getSettingsRef();
|
||||
|
||||
if (tables_name.empty())
|
||||
return tables_and_columns;
|
||||
|
||||
Block tables_columns_sample_block
|
||||
{
|
||||
{ std::make_shared<DataTypeString>(), "table_name" },
|
||||
{ std::make_shared<DataTypeString>(), "column_name" },
|
||||
{ std::make_shared<DataTypeString>(), "column_type" },
|
||||
{ std::make_shared<DataTypeUInt8>(), "is_nullable" },
|
||||
{ std::make_shared<DataTypeUInt8>(), "is_unsigned" },
|
||||
{ std::make_shared<DataTypeUInt64>(), "length" },
|
||||
};
|
||||
|
||||
WriteBufferFromOwnString query;
|
||||
query << "SELECT "
|
||||
" TABLE_NAME AS table_name,"
|
||||
" COLUMN_NAME AS column_name,"
|
||||
" DATA_TYPE AS column_type,"
|
||||
" IS_NULLABLE = 'YES' AS is_nullable,"
|
||||
" COLUMN_TYPE LIKE '%unsigned' AS is_unsigned,"
|
||||
" CHARACTER_MAXIMUM_LENGTH AS length"
|
||||
" FROM INFORMATION_SCHEMA.COLUMNS"
|
||||
" WHERE TABLE_SCHEMA = " << quote << database_name_in_mysql
|
||||
<< " AND TABLE_NAME IN " << toQueryStringWithQuote(tables_name) << " ORDER BY ORDINAL_POSITION";
|
||||
|
||||
const auto & external_table_functions_use_nulls = global_context.getSettings().external_table_functions_use_nulls;
|
||||
MySQLBlockInputStream result(mysql_pool.get(), query.str(), tables_columns_sample_block, DEFAULT_BLOCK_SIZE);
|
||||
while (Block block = result.read())
|
||||
{
|
||||
size_t rows = block.rows();
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
String table_name = (*block.getByPosition(0).column)[i].safeGet<String>();
|
||||
tables_and_columns[table_name].emplace_back((*block.getByPosition(1).column)[i].safeGet<String>(),
|
||||
convertMySQLDataType(
|
||||
(*block.getByPosition(2).column)[i].safeGet<String>(),
|
||||
(*block.getByPosition(3).column)[i].safeGet<UInt64>() &&
|
||||
external_table_functions_use_nulls,
|
||||
(*block.getByPosition(4).column)[i].safeGet<UInt64>(),
|
||||
(*block.getByPosition(5).column)[i].safeGet<UInt64>()));
|
||||
}
|
||||
}
|
||||
return tables_and_columns;
|
||||
return DB::fetchTablesColumnsList(
|
||||
mysql_pool,
|
||||
database_name_in_mysql,
|
||||
tables_name,
|
||||
settings.external_table_functions_use_nulls,
|
||||
mysql_datatypes_support_level);
|
||||
}
|
||||
|
||||
void DatabaseConnectionMySQL::shutdown()
|
||||
|
||||
@ -4,17 +4,27 @@
|
||||
#if USE_MYSQL
|
||||
|
||||
#include <mysqlxx/Pool.h>
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
#include <memory>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class Context;
|
||||
|
||||
enum class MySQLDataTypesSupport;
|
||||
|
||||
/** Real-time access to table list and table structure from remote MySQL
|
||||
* It doesn't make any manipulations with filesystem.
|
||||
* All tables are created by calling code after real-time pull-out structure from remote MySQL
|
||||
@ -25,7 +35,7 @@ public:
|
||||
~DatabaseConnectionMySQL() override;
|
||||
|
||||
DatabaseConnectionMySQL(
|
||||
const Context & global_context, const String & database_name, const String & metadata_path,
|
||||
const Context & context, const String & database_name, const String & metadata_path,
|
||||
const ASTStorage * database_engine_define, const String & database_name_in_mysql, mysqlxx::Pool && pool);
|
||||
|
||||
String getEngineName() const override { return "MySQL"; }
|
||||
@ -66,6 +76,9 @@ private:
|
||||
String metadata_path;
|
||||
ASTPtr database_engine_define;
|
||||
String database_name_in_mysql;
|
||||
// Cache setting for later from query context upon creation,
|
||||
// so column types depend on the settings set at query-level.
|
||||
MultiEnum<MySQLDataTypesSupport> mysql_datatypes_support_level;
|
||||
|
||||
std::atomic<bool> quit{false};
|
||||
std::condition_variable cond;
|
||||
@ -81,15 +94,15 @@ private:
|
||||
|
||||
void cleanOutdatedTables();
|
||||
|
||||
void fetchTablesIntoLocalCache() const;
|
||||
void fetchTablesIntoLocalCache(const Context & context) const;
|
||||
|
||||
std::map<String, UInt64> fetchTablesWithModificationTime() const;
|
||||
|
||||
std::map<String, NamesAndTypesList> fetchTablesColumnsList(const std::vector<String> & tables_name) const;
|
||||
std::map<String, NamesAndTypesList> fetchTablesColumnsList(const std::vector<String> & tables_name, const Context & context) const;
|
||||
|
||||
void destroyLocalCacheExtraTables(const std::map<String, UInt64> & tables_with_modification_time) const;
|
||||
|
||||
void fetchLatestTablesStructureIntoCache(const std::map<String, UInt64> & tables_modification_time) const;
|
||||
void fetchLatestTablesStructureIntoCache(const std::map<String, UInt64> & tables_modification_time, const Context & context) const;
|
||||
|
||||
ThreadFromGlobalPool thread;
|
||||
};
|
||||
|
||||
114
src/Databases/MySQL/FetchTablesColumnsList.cpp
Normal file
114
src/Databases/MySQL/FetchTablesColumnsList.cpp
Normal file
@ -0,0 +1,114 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_core.h"
|
||||
#endif
|
||||
|
||||
#if USE_MYSQL
|
||||
#include <Core/Block.h>
|
||||
#include <Databases/MySQL/FetchTablesColumnsList.h>
|
||||
#include <DataTypes/convertMySQLDataType.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Formats/MySQLBlockInputStream.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
String toQueryStringWithQuote(const std::vector<String> & quote_list)
|
||||
{
|
||||
WriteBufferFromOwnString quote_list_query;
|
||||
quote_list_query << "(";
|
||||
|
||||
for (size_t index = 0; index < quote_list.size(); ++index)
|
||||
{
|
||||
if (index)
|
||||
quote_list_query << ",";
|
||||
|
||||
quote_list_query << quote << quote_list[index];
|
||||
}
|
||||
|
||||
quote_list_query << ")";
|
||||
return quote_list_query.str();
|
||||
}
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
std::map<String, NamesAndTypesList> fetchTablesColumnsList(
|
||||
mysqlxx::Pool & pool,
|
||||
const String & database_name,
|
||||
const std::vector<String> & tables_name,
|
||||
bool external_table_functions_use_nulls,
|
||||
MultiEnum<MySQLDataTypesSupport> type_support)
|
||||
{
|
||||
std::map<String, NamesAndTypesList> tables_and_columns;
|
||||
|
||||
if (tables_name.empty())
|
||||
return tables_and_columns;
|
||||
|
||||
Block tables_columns_sample_block
|
||||
{
|
||||
{ std::make_shared<DataTypeString>(), "table_name" },
|
||||
{ std::make_shared<DataTypeString>(), "column_name" },
|
||||
{ std::make_shared<DataTypeString>(), "column_type" },
|
||||
{ std::make_shared<DataTypeUInt8>(), "is_nullable" },
|
||||
{ std::make_shared<DataTypeUInt8>(), "is_unsigned" },
|
||||
{ std::make_shared<DataTypeUInt64>(), "length" },
|
||||
{ std::make_shared<DataTypeUInt64>(), "precision" },
|
||||
{ std::make_shared<DataTypeUInt64>(), "scale" },
|
||||
};
|
||||
|
||||
WriteBufferFromOwnString query;
|
||||
query << "SELECT "
|
||||
" TABLE_NAME AS table_name,"
|
||||
" COLUMN_NAME AS column_name,"
|
||||
" COLUMN_TYPE AS column_type,"
|
||||
" IS_NULLABLE = 'YES' AS is_nullable,"
|
||||
" COLUMN_TYPE LIKE '%unsigned' AS is_unsigned,"
|
||||
" CHARACTER_MAXIMUM_LENGTH AS length,"
|
||||
" NUMERIC_PRECISION as '',"
|
||||
" IF(ISNULL(NUMERIC_SCALE), DATETIME_PRECISION, NUMERIC_SCALE) AS scale" // we know DATETIME_PRECISION as a scale in CH
|
||||
" FROM INFORMATION_SCHEMA.COLUMNS"
|
||||
" WHERE TABLE_SCHEMA = " << quote << database_name
|
||||
<< " AND TABLE_NAME IN " << toQueryStringWithQuote(tables_name) << " ORDER BY ORDINAL_POSITION";
|
||||
|
||||
MySQLBlockInputStream result(pool.get(), query.str(), tables_columns_sample_block, DEFAULT_BLOCK_SIZE);
|
||||
while (Block block = result.read())
|
||||
{
|
||||
const auto & table_name_col = *block.getByPosition(0).column;
|
||||
const auto & column_name_col = *block.getByPosition(1).column;
|
||||
const auto & column_type_col = *block.getByPosition(2).column;
|
||||
const auto & is_nullable_col = *block.getByPosition(3).column;
|
||||
const auto & is_unsigned_col = *block.getByPosition(4).column;
|
||||
const auto & char_max_length_col = *block.getByPosition(5).column;
|
||||
const auto & precision_col = *block.getByPosition(6).column;
|
||||
const auto & scale_col = *block.getByPosition(7).column;
|
||||
|
||||
size_t rows = block.rows();
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
String table_name = table_name_col[i].safeGet<String>();
|
||||
tables_and_columns[table_name].emplace_back(
|
||||
column_name_col[i].safeGet<String>(),
|
||||
convertMySQLDataType(
|
||||
type_support,
|
||||
column_type_col[i].safeGet<String>(),
|
||||
external_table_functions_use_nulls && is_nullable_col[i].safeGet<UInt64>(),
|
||||
is_unsigned_col[i].safeGet<UInt64>(),
|
||||
char_max_length_col[i].safeGet<UInt64>(),
|
||||
precision_col[i].safeGet<UInt64>(),
|
||||
scale_col[i].safeGet<UInt64>()));
|
||||
}
|
||||
}
|
||||
return tables_and_columns;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
28
src/Databases/MySQL/FetchTablesColumnsList.h
Normal file
28
src/Databases/MySQL/FetchTablesColumnsList.h
Normal file
@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include "config_core.h"
|
||||
#if USE_MYSQL
|
||||
|
||||
#include <mysqlxx/Pool.h>
|
||||
|
||||
#include <common/types.h>
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <Core/NamesAndTypes.h>
|
||||
#include <Core/SettingsEnums.h>
|
||||
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
std::map<String, NamesAndTypesList> fetchTablesColumnsList(
|
||||
mysqlxx::Pool & pool,
|
||||
const String & database_name,
|
||||
const std::vector<String> & tables_name,
|
||||
bool external_table_functions_use_nulls,
|
||||
MultiEnum<MySQLDataTypesSupport> type_support);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -19,6 +19,7 @@ SRCS(
|
||||
DatabaseWithDictionaries.cpp
|
||||
MySQL/DatabaseConnectionMySQL.cpp
|
||||
MySQL/DatabaseMaterializeMySQL.cpp
|
||||
MySQL/FetchTablesColumnsList.cpp
|
||||
MySQL/MaterializeMetadata.cpp
|
||||
MySQL/MaterializeMySQLSettings.cpp
|
||||
MySQL/MaterializeMySQLSyncThread.cpp
|
||||
|
||||
@ -19,6 +19,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
CassandraBlockInputStream::CassandraBlockInputStream(
|
||||
@ -140,6 +141,8 @@ void CassandraBlockInputStream::insertValue(IColumn & column, ValueType type, co
|
||||
assert_cast<ColumnUInt128 &>(column).insert(parse<UUID>(uuid_str.data(), uuid_str.size()));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw Exception("Unknown type : " + std::to_string(static_cast<int>(type)), ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
}
|
||||
|
||||
@ -252,6 +255,8 @@ void CassandraBlockInputStream::assertTypes(const CassResultPtr & result)
|
||||
expected = CASS_VALUE_TYPE_UUID;
|
||||
expected_text = "uuid";
|
||||
break;
|
||||
default:
|
||||
throw Exception("Unknown type : " + std::to_string(static_cast<int>(description.types[i].first)), ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
|
||||
CassValueType got = cass_result_column_type(result, i);
|
||||
|
||||
@ -26,6 +26,7 @@ namespace DB
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int INTERNAL_REDIS_ERROR;
|
||||
extern const int UNKNOWN_TYPE;
|
||||
}
|
||||
|
||||
|
||||
@ -103,6 +104,8 @@ namespace DB
|
||||
case ValueType::vtUUID:
|
||||
assert_cast<ColumnUInt128 &>(column).insertValue(parse<UUID>(string_value));
|
||||
break;
|
||||
default:
|
||||
throw Exception("Value of unsupported type:" + column.getName(), ErrorCodes::UNKNOWN_TYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,9 +145,12 @@ void registerDiskS3(DiskFactory & factory)
|
||||
config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024));
|
||||
|
||||
/// This code is used only to check access to the corresponding disk.
|
||||
checkWriteAccess(*s3disk);
|
||||
checkReadAccess(name, *s3disk);
|
||||
checkRemoveAccess(*s3disk);
|
||||
if (!config.getBool(config_prefix + ".skip_access_check", false))
|
||||
{
|
||||
checkWriteAccess(*s3disk);
|
||||
checkReadAccess(name, *s3disk);
|
||||
checkRemoveAccess(*s3disk);
|
||||
}
|
||||
|
||||
bool cache_enabled = config.getBool(config_prefix + ".cache_enabled", true);
|
||||
|
||||
|
||||
@ -111,6 +111,7 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co
|
||||
format_settings.template_settings.row_format = settings.format_template_row;
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line;
|
||||
format_settings.tsv.null_representation = settings.output_format_tsv_null_representation;
|
||||
format_settings.write_statistics = settings.output_format_write_statistics;
|
||||
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
|
||||
format_settings.schema.format_schema = settings.format_schema;
|
||||
|
||||
@ -78,6 +78,7 @@ struct FormatSettings
|
||||
{
|
||||
bool empty_as_default = false;
|
||||
bool crlf_end_of_line = false;
|
||||
String null_representation = "\\N";
|
||||
};
|
||||
|
||||
TSV tsv;
|
||||
|
||||
@ -7,13 +7,15 @@
|
||||
# include <Columns/ColumnNullable.h>
|
||||
# include <Columns/ColumnString.h>
|
||||
# include <Columns/ColumnsNumber.h>
|
||||
# include <Columns/ColumnDecimal.h>
|
||||
# include <DataTypes/IDataType.h>
|
||||
# include <DataTypes/DataTypeNullable.h>
|
||||
# include <IO/ReadHelpers.h>
|
||||
# include <IO/WriteHelpers.h>
|
||||
# include <Common/assert_cast.h>
|
||||
# include <ext/range.h>
|
||||
# include "MySQLBlockInputStream.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
@ -39,7 +41,7 @@ namespace
|
||||
{
|
||||
using ValueType = ExternalResultDescription::ValueType;
|
||||
|
||||
void insertValue(IColumn & column, const ValueType type, const mysqlxx::Value & value)
|
||||
void insertValue(const IDataType & data_type, IColumn & column, const ValueType type, const mysqlxx::Value & value)
|
||||
{
|
||||
switch (type)
|
||||
{
|
||||
@ -85,6 +87,15 @@ namespace
|
||||
case ValueType::vtUUID:
|
||||
assert_cast<ColumnUInt128 &>(column).insert(parse<UUID>(value.data(), value.size()));
|
||||
break;
|
||||
case ValueType::vtDateTime64:[[fallthrough]];
|
||||
case ValueType::vtDecimal32: [[fallthrough]];
|
||||
case ValueType::vtDecimal64: [[fallthrough]];
|
||||
case ValueType::vtDecimal128:
|
||||
{
|
||||
ReadBuffer buffer(const_cast<char *>(value.data()), value.size(), 0);
|
||||
data_type.deserializeAsWholeText(column, buffer, FormatSettings{});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -112,19 +123,21 @@ Block MySQLBlockInputStream::readImpl()
|
||||
for (const auto idx : ext::range(0, row.size()))
|
||||
{
|
||||
const auto value = row[idx];
|
||||
const auto & sample = description.sample_block.getByPosition(idx);
|
||||
if (!value.isNull())
|
||||
{
|
||||
if (description.types[idx].second)
|
||||
{
|
||||
ColumnNullable & column_nullable = assert_cast<ColumnNullable &>(*columns[idx]);
|
||||
insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value);
|
||||
const auto & data_type = assert_cast<const DataTypeNullable &>(*sample.type);
|
||||
insertValue(*data_type.getNestedType(), column_nullable.getNestedColumn(), description.types[idx].first, value);
|
||||
column_nullable.getNullMapData().emplace_back(0);
|
||||
}
|
||||
else
|
||||
insertValue(*columns[idx], description.types[idx].first, value);
|
||||
insertValue(*sample.type, *columns[idx], description.types[idx].first, value);
|
||||
}
|
||||
else
|
||||
insertDefaultValue(*columns[idx], *description.sample_block.getByPosition(idx).column);
|
||||
insertDefaultValue(*columns[idx], *sample.column);
|
||||
}
|
||||
|
||||
++num_rows;
|
||||
|
||||
@ -72,6 +72,9 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <class Impl>
|
||||
struct CRCFunctionWrapper
|
||||
{
|
||||
@ -127,6 +130,8 @@ using FunctionCRC32IEEE = FunctionCRC<CRC32IEEEImpl>;
|
||||
// Uses CRC-64-ECMA polynomial
|
||||
using FunctionCRC64ECMA = FunctionCRC<CRC64ECMAImpl>;
|
||||
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void registerFunctionCRCImpl(FunctionFactory & factory)
|
||||
{
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <Columns/ColumnAggregateFunction.h>
|
||||
#include "IFunctionImpl.h"
|
||||
#include "FunctionHelpers.h"
|
||||
#include "IsOperation.h"
|
||||
#include "DivisionUtils.h"
|
||||
#include "castTypeToEither.h"
|
||||
#include "FunctionFactory.h"
|
||||
@ -167,17 +168,6 @@ struct BinaryOperationImpl : BinaryOperationImplBase<A, B, Op, ResultType>
|
||||
};
|
||||
|
||||
|
||||
template <typename, typename> struct PlusImpl;
|
||||
template <typename, typename> struct MinusImpl;
|
||||
template <typename, typename> struct MultiplyImpl;
|
||||
template <typename, typename> struct DivideFloatingImpl;
|
||||
template <typename, typename> struct DivideIntegralImpl;
|
||||
template <typename, typename> struct DivideIntegralOrZeroImpl;
|
||||
template <typename, typename> struct LeastBaseImpl;
|
||||
template <typename, typename> struct GreatestBaseImpl;
|
||||
template <typename, typename> struct ModuloImpl;
|
||||
|
||||
|
||||
/// Binary operations for Decimals need scale args
|
||||
/// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2);
|
||||
/// * no agrs scale. ScaleR = Scale1 + Scale2;
|
||||
@ -185,15 +175,15 @@ template <typename, typename> struct ModuloImpl;
|
||||
template <typename A, typename B, template <typename, typename> typename Operation, typename ResultType_, bool _check_overflow = true>
|
||||
struct DecimalBinaryOperation
|
||||
{
|
||||
static constexpr bool is_plus_minus = std::is_same_v<Operation<Int32, Int32>, PlusImpl<Int32, Int32>> ||
|
||||
std::is_same_v<Operation<Int32, Int32>, MinusImpl<Int32, Int32>>;
|
||||
static constexpr bool is_multiply = std::is_same_v<Operation<Int32, Int32>, MultiplyImpl<Int32, Int32>>;
|
||||
static constexpr bool is_float_division = std::is_same_v<Operation<Int32, Int32>, DivideFloatingImpl<Int32, Int32>>;
|
||||
static constexpr bool is_int_division = std::is_same_v<Operation<Int32, Int32>, DivideIntegralImpl<Int32, Int32>> ||
|
||||
std::is_same_v<Operation<Int32, Int32>, DivideIntegralOrZeroImpl<Int32, Int32>>;
|
||||
static constexpr bool is_plus_minus = IsOperation<Operation>::plus ||
|
||||
IsOperation<Operation>::minus;
|
||||
static constexpr bool is_multiply = IsOperation<Operation>::multiply;
|
||||
static constexpr bool is_float_division = IsOperation<Operation>::div_floating;
|
||||
static constexpr bool is_int_division = IsOperation<Operation>::div_int ||
|
||||
IsOperation<Operation>::div_int_or_zero;
|
||||
static constexpr bool is_division = is_float_division || is_int_division;
|
||||
static constexpr bool is_compare = std::is_same_v<Operation<Int32, Int32>, LeastBaseImpl<Int32, Int32>> ||
|
||||
std::is_same_v<Operation<Int32, Int32>, GreatestBaseImpl<Int32, Int32>>;
|
||||
static constexpr bool is_compare = IsOperation<Operation>::least ||
|
||||
IsOperation<Operation>::greatest;
|
||||
static constexpr bool is_plus_minus_compare = is_plus_minus || is_compare;
|
||||
static constexpr bool can_overflow = is_plus_minus || is_multiply;
|
||||
|
||||
@ -529,15 +519,7 @@ private: /// it's not correct for Decimal
|
||||
using Op = Operation<T0, T1>;
|
||||
public:
|
||||
|
||||
static constexpr bool allow_decimal =
|
||||
std::is_same_v<Operation<T0, T0>, PlusImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, MinusImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, MultiplyImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, DivideFloatingImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, DivideIntegralImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, DivideIntegralOrZeroImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, LeastBaseImpl<T0, T0>> ||
|
||||
std::is_same_v<Operation<T0, T0>, GreatestBaseImpl<T0, T0>>;
|
||||
static constexpr bool allow_decimal = IsOperation<Operation>::allow_decimal;
|
||||
|
||||
/// Appropriate result type for binary operator on numeric types. "Date" can also mean
|
||||
/// DateTime, but if both operands are Dates, their type must be the same (e.g. Date - DateTime is invalid).
|
||||
@ -556,21 +538,21 @@ public:
|
||||
DataTypeFromFieldType<typename Op::ResultType>>,
|
||||
/// Date + Integral -> Date
|
||||
/// Integral + Date -> Date
|
||||
Case<std::is_same_v<Op, PlusImpl<T0, T1>>, Switch<
|
||||
Case<IsOperation<Operation>::plus, Switch<
|
||||
Case<IsIntegral<RightDataType>, LeftDataType>,
|
||||
Case<IsIntegral<LeftDataType>, RightDataType>>>,
|
||||
/// Date - Date -> Int32
|
||||
/// Date - Integral -> Date
|
||||
Case<std::is_same_v<Op, MinusImpl<T0, T1>>, Switch<
|
||||
Case<IsOperation<Operation>::minus, Switch<
|
||||
Case<std::is_same_v<LeftDataType, RightDataType>, DataTypeInt32>,
|
||||
Case<IsDateOrDateTime<LeftDataType> && IsIntegral<RightDataType>, LeftDataType>>>,
|
||||
/// least(Date, Date) -> Date
|
||||
/// greatest(Date, Date) -> Date
|
||||
Case<std::is_same_v<LeftDataType, RightDataType> && (std::is_same_v<Op, LeastBaseImpl<T0, T1>> || std::is_same_v<Op, GreatestBaseImpl<T0, T1>>),
|
||||
Case<std::is_same_v<LeftDataType, RightDataType> && (IsOperation<Operation>::least || IsOperation<Operation>::greatest),
|
||||
LeftDataType>,
|
||||
/// Date % Int32 -> Int32
|
||||
/// Date % Float -> Float64
|
||||
Case<std::is_same_v<Op, ModuloImpl<T0, T1>>, Switch<
|
||||
Case<IsOperation<Operation>::modulo, Switch<
|
||||
Case<IsDateOrDateTime<LeftDataType> && IsIntegral<RightDataType>, RightDataType>,
|
||||
Case<IsDateOrDateTime<LeftDataType> && IsFloatingPoint<RightDataType>, DataTypeFloat64>>>>;
|
||||
};
|
||||
@ -579,6 +561,9 @@ public:
|
||||
template <template <typename, typename> class Op, typename Name, bool valid_on_default_arguments = true>
|
||||
class FunctionBinaryArithmetic : public IFunction
|
||||
{
|
||||
static constexpr const bool is_multiply = IsOperation<Op>::multiply;
|
||||
static constexpr const bool is_division = IsOperation<Op>::division;
|
||||
|
||||
const Context & context;
|
||||
bool check_decimal_overflow = true;
|
||||
|
||||
@ -627,10 +612,9 @@ class FunctionBinaryArithmetic : public IFunction
|
||||
/// Special case when the function is plus or minus, one of arguments is Date/DateTime and another is Interval.
|
||||
/// We construct another function (example: addMonths) and call it.
|
||||
|
||||
static constexpr bool function_is_plus = std::is_same_v<Op<UInt8, UInt8>, PlusImpl<UInt8, UInt8>>;
|
||||
static constexpr bool function_is_minus = std::is_same_v<Op<UInt8, UInt8>, MinusImpl<UInt8, UInt8>>;
|
||||
|
||||
if (!function_is_plus && !function_is_minus)
|
||||
static constexpr bool function_is_plus = IsOperation<Op>::plus;
|
||||
static constexpr bool function_is_minus = IsOperation<Op>::minus;
|
||||
if constexpr (!function_is_plus && !function_is_minus)
|
||||
return {};
|
||||
|
||||
const DataTypePtr & type_time = first_is_date_or_datetime ? type0 : type1;
|
||||
@ -669,7 +653,7 @@ class FunctionBinaryArithmetic : public IFunction
|
||||
|
||||
bool isAggregateMultiply(const DataTypePtr & type0, const DataTypePtr & type1) const
|
||||
{
|
||||
if constexpr (!std::is_same_v<Op<UInt8, UInt8>, MultiplyImpl<UInt8, UInt8>>)
|
||||
if constexpr (!IsOperation<Op>::multiply)
|
||||
return false;
|
||||
|
||||
WhichDataType which0(type0);
|
||||
@ -681,7 +665,7 @@ class FunctionBinaryArithmetic : public IFunction
|
||||
|
||||
bool isAggregateAddition(const DataTypePtr & type0, const DataTypePtr & type1) const
|
||||
{
|
||||
if constexpr (!std::is_same_v<Op<UInt8, UInt8>, PlusImpl<UInt8, UInt8>>)
|
||||
if constexpr (!IsOperation<Op>::plus)
|
||||
return false;
|
||||
|
||||
WhichDataType which0(type0);
|
||||
@ -877,7 +861,7 @@ public:
|
||||
return false;
|
||||
else if constexpr (std::is_same_v<LeftDataType, RightDataType>)
|
||||
{
|
||||
if (left.getN() == right.getN())
|
||||
if (left.getN() == right.getN())
|
||||
{
|
||||
type_res = std::make_shared<LeftDataType>(left.getN());
|
||||
return true;
|
||||
@ -891,12 +875,7 @@ public:
|
||||
{
|
||||
if constexpr (IsDataTypeDecimal<LeftDataType> && IsDataTypeDecimal<RightDataType>)
|
||||
{
|
||||
constexpr bool is_multiply = std::is_same_v<Op<UInt8, UInt8>, MultiplyImpl<UInt8, UInt8>>;
|
||||
constexpr bool is_division = std::is_same_v<Op<UInt8, UInt8>, DivideFloatingImpl<UInt8, UInt8>> ||
|
||||
std::is_same_v<Op<UInt8, UInt8>, DivideIntegralImpl<UInt8, UInt8>> ||
|
||||
std::is_same_v<Op<UInt8, UInt8>, DivideIntegralOrZeroImpl<UInt8, UInt8>>;
|
||||
|
||||
ResultDataType result_type = decimalResultType(left, right, is_multiply, is_division);
|
||||
ResultDataType result_type = decimalResultType<is_multiply, is_division>(left, right);
|
||||
type_res = std::make_shared<ResultDataType>(result_type.getPrecision(), result_type.getScale());
|
||||
}
|
||||
else if constexpr (IsDataTypeDecimal<LeftDataType>)
|
||||
@ -920,7 +899,7 @@ public:
|
||||
type_res = std::make_shared<ResultDataType>();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
});
|
||||
if (!valid)
|
||||
@ -1016,10 +995,6 @@ public:
|
||||
if constexpr (!std::is_same_v<ResultDataType, InvalidType>)
|
||||
{
|
||||
constexpr bool result_is_decimal = IsDataTypeDecimal<LeftDataType> || IsDataTypeDecimal<RightDataType>;
|
||||
constexpr bool is_multiply = std::is_same_v<Op<UInt8, UInt8>, MultiplyImpl<UInt8, UInt8>>;
|
||||
constexpr bool is_division = std::is_same_v<Op<UInt8, UInt8>, DivideFloatingImpl<UInt8, UInt8>> ||
|
||||
std::is_same_v<Op<UInt8, UInt8>, DivideIntegralImpl<UInt8, UInt8>> ||
|
||||
std::is_same_v<Op<UInt8, UInt8>, DivideIntegralOrZeroImpl<UInt8, UInt8>>;
|
||||
|
||||
using T0 = typename LeftDataType::FieldType;
|
||||
using T1 = typename RightDataType::FieldType;
|
||||
@ -1042,7 +1017,7 @@ public:
|
||||
/// the only case with a non-vector result
|
||||
if constexpr (result_is_decimal)
|
||||
{
|
||||
ResultDataType type = decimalResultType(left, right, is_multiply, is_division);
|
||||
ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
|
||||
typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
|
||||
typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
|
||||
if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
|
||||
@ -1067,7 +1042,7 @@ public:
|
||||
typename ColVecResult::MutablePtr col_res = nullptr;
|
||||
if constexpr (result_is_decimal)
|
||||
{
|
||||
ResultDataType type = decimalResultType(left, right, is_multiply, is_division);
|
||||
ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
|
||||
col_res = ColVecResult::create(0, type.getScale());
|
||||
}
|
||||
else
|
||||
@ -1082,7 +1057,7 @@ public:
|
||||
{
|
||||
if constexpr (result_is_decimal)
|
||||
{
|
||||
ResultDataType type = decimalResultType(left, right, is_multiply, is_division);
|
||||
ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
|
||||
|
||||
typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
|
||||
typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
|
||||
@ -1102,12 +1077,13 @@ public:
|
||||
{
|
||||
if constexpr (result_is_decimal)
|
||||
{
|
||||
ResultDataType type = decimalResultType(left, right, is_multiply, is_division);
|
||||
ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
|
||||
|
||||
typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
|
||||
typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
|
||||
if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
|
||||
scale_a = right.getScaleMultiplier();
|
||||
|
||||
if (auto col_right = checkAndGetColumn<ColVecT1>(col_right_raw))
|
||||
{
|
||||
OpImpl::vectorVector(col_left->getData(), col_right->getData(), vec_res, scale_a, scale_b,
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Functions/IFunctionImpl.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IsOperation.h>
|
||||
#include <Functions/castTypeToEither.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
@ -71,9 +72,6 @@ struct FixedStringUnaryOperationImpl
|
||||
template <typename FunctionName>
|
||||
struct FunctionUnaryArithmeticMonotonicity;
|
||||
|
||||
template <typename> struct AbsImpl;
|
||||
template <typename> struct NegateImpl;
|
||||
|
||||
/// Used to indicate undefined operation
|
||||
struct InvalidType;
|
||||
|
||||
@ -81,7 +79,7 @@ struct InvalidType;
|
||||
template <template <typename> class Op, typename Name, bool is_injective>
|
||||
class FunctionUnaryArithmetic : public IFunction
|
||||
{
|
||||
static constexpr bool allow_decimal = std::is_same_v<Op<Int8>, NegateImpl<Int8>> || std::is_same_v<Op<Int8>, AbsImpl<Int8>>;
|
||||
static constexpr bool allow_decimal = IsUnaryOperation<Op>::negate || IsUnaryOperation<Op>::abs;
|
||||
static constexpr bool allow_fixed_string = Op<UInt8>::allow_fixed_string;
|
||||
|
||||
template <typename F>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user