mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into enable-merge-filters-optimization
This commit is contained in:
Nikolai Kochetov
GitHub
commit
67540a253a
@ -488,6 +488,7 @@
|
||||
* Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### New Feature
|
||||
* Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
|
||||
@ -599,7 +600,6 @@
|
||||
* Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)).
|
||||
* Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
|
||||
* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)).
|
||||
* Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# The Dockerfile.ubuntu exists for the tests/ci/docker_server.py script
|
||||
# If the image is built from Dockerfile.alpine, then the `-alpine` suffix is added automatically,
|
||||
# so the only purpose of Dockerfile.ubuntu is to push `latest`, `head` and so on w/o suffixes
|
||||
FROM ubuntu:20.04 AS glibc-donor
|
||||
FROM ubuntu:22.04 AS glibc-donor
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
@ -9,7 +9,11 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
amd64) rarch=x86_64 ;; \
|
||||
arm64) rarch=aarch64 ;; \
|
||||
esac \
|
||||
&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu
|
||||
&& ln -s "${rarch}-linux-gnu" /lib/linux-gnu \
|
||||
&& case $arch in \
|
||||
amd64) ln /lib/linux-gnu/ld-linux-x86-64.so.2 /lib/linux-gnu/ld-2.35.so ;; \
|
||||
arm64) ln /lib/linux-gnu/ld-linux-aarch64.so.1 /lib/linux-gnu/ld-2.35.so ;; \
|
||||
esac
|
||||
|
||||
|
||||
FROM alpine
|
||||
@ -20,7 +24,7 @@ ENV LANG=en_US.UTF-8 \
|
||||
TZ=UTC \
|
||||
CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml
|
||||
|
||||
COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
|
||||
COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.35.so /lib/
|
||||
COPY --from=glibc-donor /etc/nsswitch.conf /etc/
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM ubuntu:20.04
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
|
||||
# It could be removed after we move on a version 23:04+
|
||||
|
||||
@ -20,6 +20,7 @@ For more information and documentation see https://clickhouse.com/.
|
||||
|
||||
- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
|
||||
- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
|
||||
- Since the Clickhouse 24.11 Ubuntu images started using `ubuntu:22.04` as its base image. It requires docker version >= `20.10.10` containing [patch](https://github.com/moby/moby/commit/977283509f75303bc6612665a04abf76ff1d2468). As a workaround you could use `docker run [--privileged | --security-opt seccomp=unconfined]` instead, however that has security implications.
|
||||
|
||||
## How to use this image
|
||||
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
# Since right now we can't set volumes to the docker during build, we split building container in stages:
|
||||
# 1. build base container
|
||||
# 2. run base conatiner with mounted volumes
|
||||
# 3. commit container as image
|
||||
FROM ubuntu:20.04 as clickhouse-test-runner-base
|
||||
|
||||
# A volume where directory with clickhouse packages to be mounted,
|
||||
# for later installing.
|
||||
VOLUME /packages
|
||||
|
||||
CMD apt-get update ;\
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
apt install -y /packages/clickhouse-common-static_*.deb \
|
||||
/packages/clickhouse-client_*.deb \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
@ -4,9 +4,13 @@ sidebar_position: 50
|
||||
sidebar_label: EmbeddedRocksDB
|
||||
---
|
||||
|
||||
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
|
||||
|
||||
# EmbeddedRocksDB Engine
|
||||
|
||||
This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).
|
||||
<CloudNotSupportedBadge />
|
||||
|
||||
This engine allows integrating ClickHouse with [RocksDB](http://rocksdb.org/).
|
||||
|
||||
## Creating a Table {#creating-a-table}
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ sidebar_label: Prometheus protocols
|
||||
## Exposing metrics {#expose}
|
||||
|
||||
:::note
|
||||
ClickHouse Cloud does not currently support connecting to Prometheus. To be notified when this feature is supported, please contact support@clickhouse.com.
|
||||
If you are using ClickHouse Cloud, you can expose metrics to Prometheus using the [Prometheus Integration](/en/integrations/prometheus).
|
||||
:::
|
||||
|
||||
ClickHouse can expose its own metrics for scraping from Prometheus:
|
||||
|
||||
@ -65,6 +65,34 @@ sudo rm -f /etc/yum.repos.d/clickhouse.repo
|
||||
|
||||
After that follow the [install guide](../getting-started/install.md#from-rpm-packages)
|
||||
|
||||
### You Can't Run Docker Container
|
||||
|
||||
You are running a simple `docker run clickhouse/clickhouse-server` and it crashes with a stack trace similar to following:
|
||||
|
||||
```
|
||||
$ docker run -it clickhouse/clickhouse-server
|
||||
........
|
||||
2024.11.06 21:04:48.912036 [ 1 ] {} <Information> SentryWriter: Sending crash reports is disabled
|
||||
Poco::Exception. Code: 1000, e.code() = 0, System exception: cannot start thread, Stack trace (when copying this message, always include the lines below):
|
||||
|
||||
0. Poco::ThreadImpl::startImpl(Poco::SharedPtr<Poco::Runnable, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::Runnable>>) @ 0x00000000157c7b34
|
||||
1. Poco::Thread::start(Poco::Runnable&) @ 0x00000000157c8a0e
|
||||
2. BaseDaemon::initializeTerminationAndSignalProcessing() @ 0x000000000d267a14
|
||||
3. BaseDaemon::initialize(Poco::Util::Application&) @ 0x000000000d2652cb
|
||||
4. DB::Server::initialize(Poco::Util::Application&) @ 0x000000000d128b38
|
||||
5. Poco::Util::Application::run() @ 0x000000001581cfda
|
||||
6. DB::Server::run() @ 0x000000000d1288f0
|
||||
7. Poco::Util::ServerApplication::run(int, char**) @ 0x0000000015825e27
|
||||
8. mainEntryClickHouseServer(int, char**) @ 0x000000000d125b38
|
||||
9. main @ 0x0000000007ea4eee
|
||||
10. ? @ 0x00007f67ff946d90
|
||||
11. ? @ 0x00007f67ff946e40
|
||||
12. _start @ 0x00000000062e802e
|
||||
(version 24.10.1.2812 (official build))
|
||||
```
|
||||
|
||||
The reason is an old docker daemon with version lower than `20.10.10`. A way to fix it either upgrading it, or running `docker run [--privileged | --security-opt seccomp=unconfined]`. The latter has security implications.
|
||||
|
||||
## Connecting to the Server {#troubleshooting-accepts-no-connections}
|
||||
|
||||
Possible issues:
|
||||
|
||||
@ -19,7 +19,7 @@ Columns:
|
||||
- `column` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Name of a column to which access is granted.
|
||||
|
||||
- `is_partial_revoke` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Logical value. It shows whether some privileges have been revoked. Possible values:
|
||||
- `0` — The row describes a partial revoke.
|
||||
- `1` — The row describes a grant.
|
||||
- `0` — The row describes a grant.
|
||||
- `1` — The row describes a partial revoke.
|
||||
|
||||
- `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Permission is granted `WITH GRANT OPTION`, see [GRANT](../../sql-reference/statements/grant.md#granting-privilege-syntax).
|
||||
|
||||
@ -512,6 +512,8 @@ The result of operator `<` for values `d1` with underlying type `T1` and `d2` wi
|
||||
- If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared).
|
||||
- If `T1 != T2`, the result will be `T1 < T2` (type names will be compared).
|
||||
|
||||
By default `Dynamic` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings.
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
@ -535,7 +537,7 @@ SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d;
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
|
||||
```
|
||||
|
||||
```sql
|
||||
@ -557,7 +559,7 @@ Example:
|
||||
```sql
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
INSERT INTO test VALUES (1::UInt32), (1::Int64), (100::UInt32), (100::Int64);
|
||||
SELECT d, dynamicType(d) FROM test ORDER by d;
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d SETTINGS allow_suspicious_types_in_order_by=1;
|
||||
```
|
||||
|
||||
```text
|
||||
@ -570,7 +572,7 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test GROUP by d;
|
||||
SELECT d, dynamicType(d) FROM test GROUP by d SETTINGS allow_suspicious_types_in_group_by=1;
|
||||
```
|
||||
|
||||
```text
|
||||
@ -582,7 +584,7 @@ SELECT d, dynamicType(d) FROM test GROUP by d;
|
||||
└─────┴────────────────┘
|
||||
```
|
||||
|
||||
**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
|
||||
**Note:** the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
|
||||
|
||||
## Reaching the limit in number of different data types stored inside Dynamic
|
||||
|
||||
|
||||
@ -58,10 +58,10 @@ SELECT json FROM test;
|
||||
└───────────────────────────────────┘
|
||||
```
|
||||
|
||||
Using CAST from 'String':
|
||||
Using CAST from `String`:
|
||||
|
||||
```sql
|
||||
SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json;
|
||||
SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON AS json;
|
||||
```
|
||||
|
||||
```text
|
||||
@ -70,7 +70,47 @@ SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::JSON as json
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
CAST from `JSON`, named `Tuple`, `Map` and `Object('json')` to `JSON` type will be supported later.
|
||||
Using CAST from `Tuple`:
|
||||
|
||||
```sql
|
||||
SELECT (tuple(42 AS b) AS a, [1, 2, 3] AS c, 'Hello, World!' AS d)::JSON AS json;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json───────────────────────────────────────────┐
|
||||
│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Using CAST from `Map`:
|
||||
|
||||
```sql
|
||||
SELECT map('a', map('b', 42), 'c', [1,2,3], 'd', 'Hello, World!')::JSON AS json;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json───────────────────────────────────────────┐
|
||||
│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Using CAST from deprecated `Object('json')`:
|
||||
|
||||
```sql
|
||||
SELECT '{"a" : {"b" : 42},"c" : [1, 2, 3], "d" : "Hello, World!"}'::Object('json')::JSON AS json;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json───────────────────────────────────────────┐
|
||||
│ {"a":{"b":42},"c":[1,2,3],"d":"Hello, World!"} │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
:::note
|
||||
CAST from `Tuple`/`Map`/`Object('json')` to `JSON` is implemented via serializing the column into `String` column containing JSON objects and deserializing it back to `JSON` type column.
|
||||
:::
|
||||
|
||||
CAST between `JSON` types with different arguments will be supported later.
|
||||
|
||||
## Reading JSON paths as subcolumns
|
||||
|
||||
|
||||
@ -441,6 +441,8 @@ SELECT v, variantType(v) FROM test ORDER by v;
|
||||
└─────┴────────────────┘
|
||||
```
|
||||
|
||||
**Note** by default `Variant` type is not allowed in `GROUP BY`/`ORDER BY` keys, if you want to use it consider its special comparison rule and enable `allow_suspicious_types_in_group_by`/`allow_suspicious_types_in_order_by` settings.
|
||||
|
||||
## JSONExtract functions with Variant
|
||||
|
||||
All `JSONExtract*` functions support `Variant` type:
|
||||
|
||||
@ -291,7 +291,7 @@ All missed values of `expr` column will be filled sequentially and other columns
|
||||
To fill multiple columns, add `WITH FILL` modifier with optional parameters after each field name in `ORDER BY` section.
|
||||
|
||||
``` sql
|
||||
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
|
||||
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr] [STALENESS const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr] [STALENESS numeric_expr]
|
||||
[INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
|
||||
```
|
||||
|
||||
@ -300,6 +300,7 @@ When `FROM const_expr` not defined sequence of filling use minimal `expr` field
|
||||
When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`.
|
||||
When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals.
|
||||
When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type.
|
||||
When `STALENESS const_numeric_expr` is defined, the query will generate rows until the difference from the previous row in the original data exceeds `const_numeric_expr`.
|
||||
`INTERPOLATE` can be applied to columns not participating in `ORDER BY WITH FILL`. Such columns are filled based on previous fields values by applying `expr`. If `expr` is not present will repeat previous value. Omitted list will result in including all allowed columns.
|
||||
|
||||
Example of a query without `WITH FILL`:
|
||||
@ -497,6 +498,64 @@ Result:
|
||||
└────────────┴────────────┴──────────┘
|
||||
```
|
||||
|
||||
Example of a query without `STALENESS`:
|
||||
|
||||
``` sql
|
||||
SELECT number as key, 5 * number value, 'original' AS source
|
||||
FROM numbers(16) WHERE key % 5 == 0
|
||||
ORDER BY key WITH FILL;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─key─┬─value─┬─source───┐
|
||||
1. │ 0 │ 0 │ original │
|
||||
2. │ 1 │ 0 │ │
|
||||
3. │ 2 │ 0 │ │
|
||||
4. │ 3 │ 0 │ │
|
||||
5. │ 4 │ 0 │ │
|
||||
6. │ 5 │ 25 │ original │
|
||||
7. │ 6 │ 0 │ │
|
||||
8. │ 7 │ 0 │ │
|
||||
9. │ 8 │ 0 │ │
|
||||
10. │ 9 │ 0 │ │
|
||||
11. │ 10 │ 50 │ original │
|
||||
12. │ 11 │ 0 │ │
|
||||
13. │ 12 │ 0 │ │
|
||||
14. │ 13 │ 0 │ │
|
||||
15. │ 14 │ 0 │ │
|
||||
16. │ 15 │ 75 │ original │
|
||||
└─────┴───────┴──────────┘
|
||||
```
|
||||
|
||||
Same query after applying `STALENESS 3`:
|
||||
|
||||
``` sql
|
||||
SELECT number as key, 5 * number value, 'original' AS source
|
||||
FROM numbers(16) WHERE key % 5 == 0
|
||||
ORDER BY key WITH FILL STALENESS 3;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─key─┬─value─┬─source───┐
|
||||
1. │ 0 │ 0 │ original │
|
||||
2. │ 1 │ 0 │ │
|
||||
3. │ 2 │ 0 │ │
|
||||
4. │ 5 │ 25 │ original │
|
||||
5. │ 6 │ 0 │ │
|
||||
6. │ 7 │ 0 │ │
|
||||
7. │ 10 │ 50 │ original │
|
||||
8. │ 11 │ 0 │ │
|
||||
9. │ 12 │ 0 │ │
|
||||
10. │ 15 │ 75 │ original │
|
||||
11. │ 16 │ 0 │ │
|
||||
12. │ 17 │ 0 │ │
|
||||
└─────┴───────┴──────────┘
|
||||
```
|
||||
|
||||
Example of a query without `INTERPOLATE`:
|
||||
|
||||
``` sql
|
||||
|
||||
@ -387,7 +387,7 @@ template <typename Value, bool return_float, bool interpolated>
|
||||
using FuncQuantileExactWeighted = AggregateFunctionQuantile<
|
||||
Value,
|
||||
QuantileExactWeighted<Value, interpolated>,
|
||||
NameQuantileExactWeighted,
|
||||
std::conditional_t<interpolated, NameQuantileExactWeightedInterpolated, NameQuantileExactWeighted>,
|
||||
true,
|
||||
std::conditional_t<return_float, Float64, void>,
|
||||
false,
|
||||
@ -396,7 +396,7 @@ template <typename Value, bool return_float, bool interpolated>
|
||||
using FuncQuantilesExactWeighted = AggregateFunctionQuantile<
|
||||
Value,
|
||||
QuantileExactWeighted<Value, interpolated>,
|
||||
NameQuantilesExactWeighted,
|
||||
std::conditional_t<interpolated, NameQuantilesExactWeightedInterpolated, NameQuantilesExactWeighted>,
|
||||
true,
|
||||
std::conditional_t<return_float, Float64, void>,
|
||||
true,
|
||||
|
||||
@ -602,9 +602,21 @@ public:
|
||||
return projection_columns;
|
||||
}
|
||||
|
||||
/// Returns true if query node is resolved, false otherwise
|
||||
bool isResolved() const
|
||||
{
|
||||
return !projection_columns.empty();
|
||||
}
|
||||
|
||||
/// Resolve query node projection columns
|
||||
void resolveProjectionColumns(NamesAndTypes projection_columns_value);
|
||||
|
||||
/// Clear query node projection columns
|
||||
void clearProjectionColumns()
|
||||
{
|
||||
projection_columns.clear();
|
||||
}
|
||||
|
||||
/// Remove unused projection columns
|
||||
void removeUnusedProjectionColumns(const std::unordered_set<std::string> & used_projection_columns);
|
||||
|
||||
|
||||
@ -498,6 +498,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express
|
||||
sort_node->getFillTo() = buildExpression(order_by_element.getFillTo(), context);
|
||||
if (order_by_element.getFillStep())
|
||||
sort_node->getFillStep() = buildExpression(order_by_element.getFillStep(), context);
|
||||
if (order_by_element.getFillStaleness())
|
||||
sort_node->getFillStaleness() = buildExpression(order_by_element.getFillStaleness(), context);
|
||||
|
||||
list_node->getNodes().push_back(std::move(sort_node));
|
||||
}
|
||||
|
||||
@ -103,6 +103,8 @@ namespace Setting
|
||||
extern const SettingsBool single_join_prefer_left_table;
|
||||
extern const SettingsBool transform_null_in;
|
||||
extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions;
|
||||
extern const SettingsBool allow_suspicious_types_in_group_by;
|
||||
extern const SettingsBool allow_suspicious_types_in_order_by;
|
||||
extern const SettingsBool use_concurrency_control;
|
||||
}
|
||||
|
||||
@ -437,8 +439,13 @@ ProjectionName QueryAnalyzer::calculateWindowProjectionName(const QueryTreeNodeP
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeNodePtr & sort_column_node, const ProjectionName & sort_expression_projection_name,
|
||||
const ProjectionName & fill_from_expression_projection_name, const ProjectionName & fill_to_expression_projection_name, const ProjectionName & fill_step_expression_projection_name)
|
||||
ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(
|
||||
const QueryTreeNodePtr & sort_column_node,
|
||||
const ProjectionName & sort_expression_projection_name,
|
||||
const ProjectionName & fill_from_expression_projection_name,
|
||||
const ProjectionName & fill_to_expression_projection_name,
|
||||
const ProjectionName & fill_step_expression_projection_name,
|
||||
const ProjectionName & fill_staleness_expression_projection_name)
|
||||
{
|
||||
auto & sort_node_typed = sort_column_node->as<SortNode &>();
|
||||
|
||||
@ -468,6 +475,9 @@ ProjectionName QueryAnalyzer::calculateSortColumnProjectionName(const QueryTreeN
|
||||
|
||||
if (sort_node_typed.hasFillStep())
|
||||
sort_column_projection_name_buffer << " STEP " << fill_step_expression_projection_name;
|
||||
|
||||
if (sort_node_typed.hasFillStaleness())
|
||||
sort_column_projection_name_buffer << " STALENESS " << fill_staleness_expression_projection_name;
|
||||
}
|
||||
|
||||
return sort_column_projection_name_buffer.str();
|
||||
@ -2958,27 +2968,29 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
||||
/// Replace storage with values storage of insertion block
|
||||
if (StoragePtr storage = scope.context->getViewSource())
|
||||
{
|
||||
QueryTreeNodePtr table_expression;
|
||||
/// Process possibly nested sub-selects
|
||||
for (auto * query_node = in_second_argument->as<QueryNode>(); query_node; query_node = table_expression->as<QueryNode>())
|
||||
table_expression = extractLeftTableExpression(query_node->getJoinTree());
|
||||
QueryTreeNodePtr table_expression = in_second_argument;
|
||||
|
||||
if (table_expression)
|
||||
/// Process possibly nested sub-selects
|
||||
while (table_expression)
|
||||
{
|
||||
if (auto * query_table_node = table_expression->as<TableNode>())
|
||||
{
|
||||
if (query_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted())
|
||||
{
|
||||
auto replacement_table_expression = std::make_shared<TableNode>(storage, scope.context);
|
||||
if (std::optional<TableExpressionModifiers> table_expression_modifiers = query_table_node->getTableExpressionModifiers())
|
||||
replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers);
|
||||
in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression));
|
||||
}
|
||||
}
|
||||
if (auto * query_node = table_expression->as<QueryNode>())
|
||||
table_expression = extractLeftTableExpression(query_node->getJoinTree());
|
||||
else if (auto * union_node = table_expression->as<UnionNode>())
|
||||
table_expression = union_node->getQueries().getNodes().at(0);
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
TableNode * table_expression_table_node = table_expression ? table_expression->as<TableNode>() : nullptr;
|
||||
|
||||
if (table_expression_table_node &&
|
||||
table_expression_table_node->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted())
|
||||
{
|
||||
auto replacement_table_expression_table_node = table_expression_table_node->clone();
|
||||
replacement_table_expression_table_node->as<TableNode &>().updateStorage(storage, scope.context);
|
||||
in_second_argument = in_second_argument->cloneAndReplace(table_expression, std::move(replacement_table_expression_table_node));
|
||||
}
|
||||
}
|
||||
|
||||
resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/);
|
||||
}
|
||||
|
||||
/// Edge case when the first argument of IN is scalar subquery.
|
||||
@ -3998,6 +4010,7 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
|
||||
ProjectionNames fill_from_expression_projection_names;
|
||||
ProjectionNames fill_to_expression_projection_names;
|
||||
ProjectionNames fill_step_expression_projection_names;
|
||||
ProjectionNames fill_staleness_expression_projection_names;
|
||||
|
||||
auto & sort_node_list_typed = sort_node_list->as<ListNode &>();
|
||||
for (auto & node : sort_node_list_typed.getNodes())
|
||||
@ -4019,6 +4032,8 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
|
||||
sort_node.getExpression() = sort_column_list_node->getNodes().front();
|
||||
}
|
||||
|
||||
validateSortingKeyType(sort_node.getExpression()->getResultType(), scope);
|
||||
|
||||
size_t sort_expression_projection_names_size = sort_expression_projection_names.size();
|
||||
if (sort_expression_projection_names_size != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
@ -4088,11 +4103,38 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
|
||||
fill_step_expression_projection_names_size);
|
||||
}
|
||||
|
||||
if (sort_node.hasFillStaleness())
|
||||
{
|
||||
fill_staleness_expression_projection_names = resolveExpressionNode(sort_node.getFillStaleness(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
|
||||
const auto * constant_node = sort_node.getFillStaleness()->as<ConstantNode>();
|
||||
if (!constant_node)
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}",
|
||||
sort_node.getFillStaleness()->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
|
||||
bool is_number = isColumnedAsNumber(constant_node->getResultType());
|
||||
bool is_interval = WhichDataType(constant_node->getResultType()).isInterval();
|
||||
if (!is_number && !is_interval)
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"Sort FILL STALENESS expression must be constant with numeric or interval type. Actual {}. In scope {}",
|
||||
sort_node.getFillStaleness()->formatASTForErrorMessage(),
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
|
||||
size_t fill_staleness_expression_projection_names_size = fill_staleness_expression_projection_names.size();
|
||||
if (fill_staleness_expression_projection_names_size != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Sort FILL STALENESS expression expected 1 projection name. Actual {}",
|
||||
fill_staleness_expression_projection_names_size);
|
||||
}
|
||||
|
||||
auto sort_column_projection_name = calculateSortColumnProjectionName(node,
|
||||
sort_expression_projection_names[0],
|
||||
fill_from_expression_projection_names.empty() ? "" : fill_from_expression_projection_names.front(),
|
||||
fill_to_expression_projection_names.empty() ? "" : fill_to_expression_projection_names.front(),
|
||||
fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front());
|
||||
fill_step_expression_projection_names.empty() ? "" : fill_step_expression_projection_names.front(),
|
||||
fill_staleness_expression_projection_names.empty() ? "" : fill_staleness_expression_projection_names.front());
|
||||
|
||||
result_projection_names.push_back(std::move(sort_column_projection_name));
|
||||
|
||||
@ -4100,11 +4142,32 @@ ProjectionNames QueryAnalyzer::resolveSortNodeList(QueryTreeNodePtr & sort_node_
|
||||
fill_from_expression_projection_names.clear();
|
||||
fill_to_expression_projection_names.clear();
|
||||
fill_step_expression_projection_names.clear();
|
||||
fill_staleness_expression_projection_names.clear();
|
||||
}
|
||||
|
||||
return result_projection_names;
|
||||
}
|
||||
|
||||
void QueryAnalyzer::validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const
|
||||
{
|
||||
if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_order_by])
|
||||
return;
|
||||
|
||||
auto check = [](const IDataType & type)
|
||||
{
|
||||
if (isDynamic(type) || isVariant(type))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_COLUMN,
|
||||
"Data types Variant/Dynamic are not allowed in ORDER BY keys, because it can lead to unexpected results. "
|
||||
"Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
|
||||
"its a JSON path subcolumn) or casting this column to a specific data type. "
|
||||
"Set setting allow_suspicious_types_in_order_by = 1 in order to allow it");
|
||||
};
|
||||
|
||||
check(*sorting_key_type);
|
||||
sorting_key_type->forEachChild(check);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
@ -4144,11 +4207,12 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR
|
||||
expandTuplesInList(group_by_list);
|
||||
}
|
||||
|
||||
if (scope.group_by_use_nulls)
|
||||
for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes())
|
||||
{
|
||||
for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes())
|
||||
for (const auto & group_by_elem : grouping_set->as<ListNode>()->getNodes())
|
||||
{
|
||||
for (const auto & group_by_elem : grouping_set->as<ListNode>()->getNodes())
|
||||
validateGroupByKeyType(group_by_elem->getResultType(), scope);
|
||||
if (scope.group_by_use_nulls)
|
||||
scope.nullable_group_by_keys.insert(group_by_elem);
|
||||
}
|
||||
}
|
||||
@ -4164,14 +4228,37 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR
|
||||
auto & group_by_list = query_node_typed.getGroupBy().getNodes();
|
||||
expandTuplesInList(group_by_list);
|
||||
|
||||
if (scope.group_by_use_nulls)
|
||||
for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes())
|
||||
{
|
||||
for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes())
|
||||
validateGroupByKeyType(group_by_elem->getResultType(), scope);
|
||||
if (scope.group_by_use_nulls)
|
||||
scope.nullable_group_by_keys.insert(group_by_elem);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Validate data types of GROUP BY key.
|
||||
*/
|
||||
void QueryAnalyzer::validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const
|
||||
{
|
||||
if (scope.context->getSettingsRef()[Setting::allow_suspicious_types_in_group_by])
|
||||
return;
|
||||
|
||||
auto check = [](const IDataType & type)
|
||||
{
|
||||
if (isDynamic(type) || isVariant(type))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_COLUMN,
|
||||
"Data types Variant/Dynamic are not allowed in GROUP BY keys, because it can lead to unexpected results. "
|
||||
"Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
|
||||
"its a JSON path subcolumn) or casting this column to a specific data type. "
|
||||
"Set setting allow_suspicious_types_in_group_by = 1 in order to allow it");
|
||||
};
|
||||
|
||||
check(*group_by_key_type);
|
||||
group_by_key_type->forEachChild(check);
|
||||
}
|
||||
|
||||
/** Resolve interpolate columns nodes list.
|
||||
*/
|
||||
void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope)
|
||||
@ -5310,6 +5397,16 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
|
||||
auto & query_node_typed = query_node->as<QueryNode &>();
|
||||
|
||||
/** It is unsafe to call resolveQuery on already resolved query node, because during identifier resolution process
|
||||
* we replace identifiers with expressions without aliases, also at the end of resolveQuery all aliases from all nodes will be removed.
|
||||
* For subsequent resolveQuery executions it is possible to have wrong projection header, because for nodes
|
||||
* with aliases projection name is alias.
|
||||
*
|
||||
* If for client it is necessary to resolve query node after clone, client must clear projection columns from query node before resolve.
|
||||
*/
|
||||
if (query_node_typed.isResolved())
|
||||
return;
|
||||
|
||||
if (query_node_typed.isCTE())
|
||||
ctes_in_resolve_process.insert(query_node_typed.getCTEName());
|
||||
|
||||
@ -5448,16 +5545,13 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
*/
|
||||
scope.use_identifier_lookup_to_result_cache = false;
|
||||
|
||||
if (query_node_typed.getJoinTree())
|
||||
{
|
||||
TableExpressionsAliasVisitor table_expressions_visitor(scope);
|
||||
table_expressions_visitor.visit(query_node_typed.getJoinTree());
|
||||
TableExpressionsAliasVisitor table_expressions_visitor(scope);
|
||||
table_expressions_visitor.visit(query_node_typed.getJoinTree());
|
||||
|
||||
initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope);
|
||||
scope.aliases.alias_name_to_table_expression_node.clear();
|
||||
initializeQueryJoinTreeNode(query_node_typed.getJoinTree(), scope);
|
||||
scope.aliases.alias_name_to_table_expression_node.clear();
|
||||
|
||||
resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor);
|
||||
}
|
||||
resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor);
|
||||
|
||||
if (!scope.group_by_use_nulls)
|
||||
scope.use_identifier_lookup_to_result_cache = true;
|
||||
@ -5675,6 +5769,9 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier
|
||||
{
|
||||
auto & union_node_typed = union_node->as<UnionNode &>();
|
||||
|
||||
if (union_node_typed.isResolved())
|
||||
return;
|
||||
|
||||
if (union_node_typed.isCTE())
|
||||
ctes_in_resolve_process.insert(union_node_typed.getCTEName());
|
||||
|
||||
|
||||
@ -140,7 +140,8 @@ private:
|
||||
const ProjectionName & sort_expression_projection_name,
|
||||
const ProjectionName & fill_from_expression_projection_name,
|
||||
const ProjectionName & fill_to_expression_projection_name,
|
||||
const ProjectionName & fill_step_expression_projection_name);
|
||||
const ProjectionName & fill_step_expression_projection_name,
|
||||
const ProjectionName & fill_staleness_expression_projection_name);
|
||||
|
||||
QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context);
|
||||
|
||||
@ -219,8 +220,12 @@ private:
|
||||
|
||||
ProjectionNames resolveSortNodeList(QueryTreeNodePtr & sort_node_list, IdentifierResolveScope & scope);
|
||||
|
||||
void validateSortingKeyType(const DataTypePtr & sorting_key_type, const IdentifierResolveScope & scope) const;
|
||||
|
||||
void resolveGroupByNode(QueryNode & query_node_typed, IdentifierResolveScope & scope);
|
||||
|
||||
void validateGroupByKeyType(const DataTypePtr & group_by_key_type, const IdentifierResolveScope & scope) const;
|
||||
|
||||
void resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpolate_node_list, IdentifierResolveScope & scope);
|
||||
|
||||
void resolveWindowNodeList(QueryTreeNodePtr & window_node_list, IdentifierResolveScope & scope);
|
||||
|
||||
@ -69,6 +69,12 @@ void SortNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, si
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "FILL STEP\n";
|
||||
getFillStep()->dumpTreeImpl(buffer, format_state, indent + 4);
|
||||
}
|
||||
|
||||
if (hasFillStaleness())
|
||||
{
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "FILL STALENESS\n";
|
||||
getFillStaleness()->dumpTreeImpl(buffer, format_state, indent + 4);
|
||||
}
|
||||
}
|
||||
|
||||
bool SortNode::isEqualImpl(const IQueryTreeNode & rhs, CompareOptions) const
|
||||
@ -132,6 +138,8 @@ ASTPtr SortNode::toASTImpl(const ConvertToASTOptions & options) const
|
||||
result->setFillTo(getFillTo()->toAST(options));
|
||||
if (hasFillStep())
|
||||
result->setFillStep(getFillStep()->toAST(options));
|
||||
if (hasFillStaleness())
|
||||
result->setFillStaleness(getFillStaleness()->toAST(options));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -105,6 +105,24 @@ public:
|
||||
return children[fill_step_child_index];
|
||||
}
|
||||
|
||||
/// Returns true if sort node has fill staleness, false otherwise
|
||||
bool hasFillStaleness() const
|
||||
{
|
||||
return children[fill_staleness_child_index] != nullptr;
|
||||
}
|
||||
|
||||
/// Get fill staleness
|
||||
const QueryTreeNodePtr & getFillStaleness() const
|
||||
{
|
||||
return children[fill_staleness_child_index];
|
||||
}
|
||||
|
||||
/// Get fill staleness
|
||||
QueryTreeNodePtr & getFillStaleness()
|
||||
{
|
||||
return children[fill_staleness_child_index];
|
||||
}
|
||||
|
||||
/// Get collator
|
||||
const std::shared_ptr<Collator> & getCollator() const
|
||||
{
|
||||
@ -144,7 +162,8 @@ private:
|
||||
static constexpr size_t fill_from_child_index = 1;
|
||||
static constexpr size_t fill_to_child_index = 2;
|
||||
static constexpr size_t fill_step_child_index = 3;
|
||||
static constexpr size_t children_size = fill_step_child_index + 1;
|
||||
static constexpr size_t fill_staleness_child_index = 4;
|
||||
static constexpr size_t children_size = fill_staleness_child_index + 1;
|
||||
|
||||
SortDirection sort_direction = SortDirection::ASCENDING;
|
||||
std::optional<SortDirection> nulls_sort_direction;
|
||||
|
||||
@ -35,6 +35,7 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int TYPE_MISMATCH;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_)
|
||||
@ -50,6 +51,26 @@ UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_)
|
||||
children[queries_child_index] = std::make_shared<ListNode>();
|
||||
}
|
||||
|
||||
bool UnionNode::isResolved() const
|
||||
{
|
||||
for (const auto & query_node : getQueries().getNodes())
|
||||
{
|
||||
bool is_resolved = false;
|
||||
|
||||
if (auto * query_node_typed = query_node->as<QueryNode>())
|
||||
is_resolved = query_node_typed->isResolved();
|
||||
else if (auto * union_node_typed = query_node->as<UnionNode>())
|
||||
is_resolved = union_node_typed->isResolved();
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected query tree node type in UNION node");
|
||||
|
||||
if (!is_resolved)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
NamesAndTypes UnionNode::computeProjectionColumns() const
|
||||
{
|
||||
if (recursive_cte_table)
|
||||
|
||||
@ -163,6 +163,9 @@ public:
|
||||
return children[queries_child_index];
|
||||
}
|
||||
|
||||
/// Returns true if union node is resolved, false otherwise
|
||||
bool isResolved() const;
|
||||
|
||||
/// Compute union node projection columns
|
||||
NamesAndTypes computeProjectionColumns() const;
|
||||
|
||||
|
||||
@ -952,7 +952,7 @@ ColumnPtr ColumnVariant::permute(const Permutation & perm, size_t limit) const
|
||||
if (hasOnlyNulls())
|
||||
{
|
||||
if (limit)
|
||||
return cloneResized(limit);
|
||||
return cloneResized(limit ? std::min(size(), limit) : size());
|
||||
|
||||
/// If no limit, we can just return current immutable column.
|
||||
return this->getPtr();
|
||||
|
||||
30
src/Common/FieldVisitorScale.cpp
Normal file
30
src/Common/FieldVisitorScale.cpp
Normal file
@ -0,0 +1,30 @@
|
||||
#include <Common/FieldVisitorScale.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
FieldVisitorScale::FieldVisitorScale(Int32 rhs_) : rhs(rhs_) {}
|
||||
|
||||
void FieldVisitorScale::operator() (Int64 & x) const { x *= rhs; }
|
||||
void FieldVisitorScale::operator() (UInt64 & x) const { x *= rhs; }
|
||||
void FieldVisitorScale::operator() (Float64 & x) const { x *= rhs; }
|
||||
void FieldVisitorScale::operator() (Null &) const { /*Do not scale anything*/ }
|
||||
|
||||
void FieldVisitorScale::operator() (String &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Strings"); }
|
||||
void FieldVisitorScale::operator() (Array &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Arrays"); }
|
||||
void FieldVisitorScale::operator() (Tuple &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Tuples"); }
|
||||
void FieldVisitorScale::operator() (Map &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Maps"); }
|
||||
void FieldVisitorScale::operator() (Object &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Objects"); }
|
||||
void FieldVisitorScale::operator() (UUID &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale UUIDs"); }
|
||||
void FieldVisitorScale::operator() (IPv4 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv4s"); }
|
||||
void FieldVisitorScale::operator() (IPv6 &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale IPv6s"); }
|
||||
void FieldVisitorScale::operator() (CustomType & x) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale custom type {}", x.getTypeName()); }
|
||||
void FieldVisitorScale::operator() (AggregateFunctionStateData &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale AggregateFunctionStates"); }
|
||||
void FieldVisitorScale::operator() (bool &) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot scale Bools"); }
|
||||
|
||||
}
|
||||
43
src/Common/FieldVisitorScale.h
Normal file
43
src/Common/FieldVisitorScale.h
Normal file
@ -0,0 +1,43 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/FieldVisitorConvertToNumber.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Implements `*=` operation by number
|
||||
*/
|
||||
class FieldVisitorScale : public StaticVisitor<void>
|
||||
{
|
||||
private:
|
||||
Int32 rhs;
|
||||
|
||||
public:
|
||||
explicit FieldVisitorScale(Int32 rhs_);
|
||||
|
||||
void operator() (Int64 & x) const;
|
||||
void operator() (UInt64 & x) const;
|
||||
void operator() (Float64 & x) const;
|
||||
void operator() (Null &) const;
|
||||
[[noreturn]] void operator() (String &) const;
|
||||
[[noreturn]] void operator() (Array &) const;
|
||||
[[noreturn]] void operator() (Tuple &) const;
|
||||
[[noreturn]] void operator() (Map &) const;
|
||||
[[noreturn]] void operator() (Object &) const;
|
||||
[[noreturn]] void operator() (UUID &) const;
|
||||
[[noreturn]] void operator() (IPv4 &) const;
|
||||
[[noreturn]] void operator() (IPv6 &) const;
|
||||
[[noreturn]] void operator() (AggregateFunctionStateData &) const;
|
||||
[[noreturn]] void operator() (CustomType &) const;
|
||||
[[noreturn]] void operator() (bool &) const;
|
||||
|
||||
template <typename T>
|
||||
void operator() (DecimalField<T> & x) const { x = DecimalField<T>(x.getValue() * T(rhs), x.getScale()); }
|
||||
|
||||
template <typename T>
|
||||
requires is_big_int_v<T>
|
||||
void operator() (T & x) const { x *= rhs; }
|
||||
};
|
||||
|
||||
}
|
||||
@ -658,16 +658,11 @@ protected:
|
||||
{
|
||||
if (!std::is_trivially_destructible_v<Cell>)
|
||||
{
|
||||
for (iterator it = begin(), it_end = end(); it != it_end; ++it)
|
||||
for (iterator it = begin(), it_end = end(); it != it_end;)
|
||||
{
|
||||
it.ptr->~Cell();
|
||||
/// In case of poison_in_dtor=1 it will be poisoned,
|
||||
/// but it maybe used later, during iteration.
|
||||
///
|
||||
/// NOTE, that technically this is UB [1], but OK for now.
|
||||
///
|
||||
/// [1]: https://github.com/google/sanitizers/issues/854#issuecomment-329661378
|
||||
__msan_unpoison(it.ptr, sizeof(*it.ptr));
|
||||
auto ptr = it.ptr;
|
||||
++it;
|
||||
ptr->~Cell();
|
||||
}
|
||||
|
||||
/// Everything had been destroyed in the loop above, reset the flag
|
||||
|
||||
@ -746,6 +746,12 @@ The server successfully detected this situation and will download merged part fr
|
||||
M(ReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.", ValueType::Microseconds) \
|
||||
M(MergeTreeReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \
|
||||
M(MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds, "Time spent in sending the announcement from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.", ValueType::Microseconds) \
|
||||
M(MergerMutatorsGetPartsForMergeElapsedMicroseconds, "Time spent to take data parts snapshot to build ranges from them.", ValueType::Microseconds) \
|
||||
M(MergerMutatorPrepareRangesForMergeElapsedMicroseconds, "Time spent to prepare parts ranges which can be merged according to merge predicate.", ValueType::Microseconds) \
|
||||
M(MergerMutatorSelectPartsForMergeElapsedMicroseconds, "Time spent to select parts from ranges which can be merged.", ValueType::Microseconds) \
|
||||
M(MergerMutatorRangesForMergeCount, "Amount of candidate ranges for merge", ValueType::Number) \
|
||||
M(MergerMutatorPartsInRangesForMergeCount, "Amount of candidate parts for merge", ValueType::Number) \
|
||||
M(MergerMutatorSelectRangePartsCount, "Amount of parts in selected range for merge", ValueType::Number) \
|
||||
\
|
||||
M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \
|
||||
M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \
|
||||
|
||||
@ -119,15 +119,4 @@ enum class JoinTableSide : uint8_t
|
||||
|
||||
const char * toString(JoinTableSide join_table_side);
|
||||
|
||||
/// Setting to choose which table to use as the inner table in hash join
|
||||
enum class JoinInnerTableSelectionMode : uint8_t
|
||||
{
|
||||
/// Use left table
|
||||
Left,
|
||||
/// Use right table
|
||||
Right,
|
||||
/// Use the table with the smallest number of rows
|
||||
Auto,
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -873,6 +873,12 @@ In CREATE TABLE statement allows specifying Variant type with similar variant ty
|
||||
)", 0) \
|
||||
DECLARE(Bool, allow_suspicious_primary_key, false, R"(
|
||||
Allow suspicious `PRIMARY KEY`/`ORDER BY` for MergeTree (i.e. SimpleAggregateFunction).
|
||||
)", 0) \
|
||||
DECLARE(Bool, allow_suspicious_types_in_group_by, false, R"(
|
||||
Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in GROUP BY keys.
|
||||
)", 0) \
|
||||
DECLARE(Bool, allow_suspicious_types_in_order_by, false, R"(
|
||||
Allows or restricts using [Variant](../../sql-reference/data-types/variant.md) and [Dynamic](../../sql-reference/data-types/dynamic.md) types in ORDER BY keys.
|
||||
)", 0) \
|
||||
DECLARE(Bool, compile_expressions, false, R"(
|
||||
Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting.
|
||||
@ -1912,9 +1918,6 @@ See also:
|
||||
For single JOIN in case of identifier ambiguity prefer left table
|
||||
)", IMPORTANT) \
|
||||
\
|
||||
DECLARE(JoinInnerTableSelectionMode, query_plan_join_inner_table_selection, JoinInnerTableSelectionMode::Auto, R"(
|
||||
Select the side of the join to be the inner table in the query plan. Supported only for `ALL` join strictness with `JOIN ON` clause. Possible values: 'auto', 'left', 'right'.
|
||||
)", 0) \
|
||||
DECLARE(UInt64, preferred_block_size_bytes, 1000000, R"(
|
||||
This setting adjusts the data block size for query processing and represents additional fine-tuning to the more rough 'max_block_size' setting. If the columns are large and with 'max_block_size' rows the block size is likely to be larger than the specified amount of bytes, its size will be lowered for better CPU cache locality.
|
||||
)", 0) \
|
||||
@ -2866,7 +2869,7 @@ Limit on size of multipart/form-data content. This setting cannot be parsed from
|
||||
DECLARE(Bool, calculate_text_stack_trace, true, R"(
|
||||
Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when a huge amount of wrong queries are executed. In normal cases, you should not disable this option.
|
||||
)", 0) \
|
||||
DECLARE(Bool, enable_job_stack_trace, false, R"(
|
||||
DECLARE(Bool, enable_job_stack_trace, true, R"(
|
||||
Output stack trace of a job creator when job results in exception
|
||||
)", 0) \
|
||||
DECLARE(Bool, allow_ddl, true, R"(
|
||||
@ -4239,7 +4242,7 @@ Rewrite aggregate functions with if expression as argument when logically equiva
|
||||
For example, `avg(if(cond, col, null))` can be rewritten to `avgOrNullIf(cond, col)`. It may improve performance.
|
||||
|
||||
:::note
|
||||
Supported only with experimental analyzer (`enable_analyzer = 1`).
|
||||
Supported only with the analyzer (`enable_analyzer = 1`).
|
||||
:::
|
||||
)", 0) \
|
||||
DECLARE(Bool, optimize_rewrite_array_exists_to_has, false, R"(
|
||||
|
||||
@ -66,7 +66,6 @@ class WriteBuffer;
|
||||
M(CLASS_NAME, IntervalOutputFormat) \
|
||||
M(CLASS_NAME, JoinAlgorithm) \
|
||||
M(CLASS_NAME, JoinStrictness) \
|
||||
M(CLASS_NAME, JoinInnerTableSelectionMode) \
|
||||
M(CLASS_NAME, LightweightMutationProjectionMode) \
|
||||
M(CLASS_NAME, LoadBalancing) \
|
||||
M(CLASS_NAME, LocalFSReadMethod) \
|
||||
|
||||
@ -64,6 +64,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
||||
},
|
||||
{"24.11",
|
||||
{
|
||||
{"allow_suspicious_types_in_group_by", true, false, "Don't allow Variant/Dynamic types in GROUP BY by default"},
|
||||
{"allow_suspicious_types_in_order_by", true, false, "Don't allow Variant/Dynamic types in ORDER BY by default"},
|
||||
{"distributed_cache_discard_connection_if_unread_data", true, true, "New setting"},
|
||||
{"filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage", true, true, "New setting"},
|
||||
{"filesystem_cache_enable_background_download_during_fetch", true, true, "New setting"},
|
||||
@ -80,6 +82,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
||||
},
|
||||
{"24.10",
|
||||
{
|
||||
{"enable_job_stack_trace", false, true, "Enable by default collecting stack traces from job's scheduling."},
|
||||
{"query_metric_log_interval", 0, -1, "New setting."},
|
||||
{"enforce_strict_identifier_format", false, false, "New setting."},
|
||||
{"enable_parsing_to_custom_serialization", false, true, "New setting"},
|
||||
|
||||
@ -55,10 +55,6 @@ IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN,
|
||||
{"full_sorting_merge", JoinAlgorithm::FULL_SORTING_MERGE},
|
||||
{"grace_hash", JoinAlgorithm::GRACE_HASH}})
|
||||
|
||||
IMPLEMENT_SETTING_ENUM(JoinInnerTableSelectionMode, ErrorCodes::BAD_ARGUMENTS,
|
||||
{{"left", JoinInnerTableSelectionMode::Left},
|
||||
{"right", JoinInnerTableSelectionMode::Right},
|
||||
{"auto", JoinInnerTableSelectionMode::Auto}})
|
||||
|
||||
IMPLEMENT_SETTING_ENUM(TotalsMode, ErrorCodes::UNKNOWN_TOTALS_MODE,
|
||||
{{"before_having", TotalsMode::BEFORE_HAVING},
|
||||
|
||||
@ -128,8 +128,8 @@ constexpr auto getEnumValues();
|
||||
DECLARE_SETTING_ENUM(LoadBalancing)
|
||||
|
||||
DECLARE_SETTING_ENUM(JoinStrictness)
|
||||
|
||||
DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm)
|
||||
DECLARE_SETTING_ENUM(JoinInnerTableSelectionMode)
|
||||
|
||||
|
||||
/// Which rows should be included in TOTALS.
|
||||
|
||||
@ -35,6 +35,11 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
/** Cursor allows to compare rows in different blocks (and parts).
|
||||
* Cursor moves inside single block.
|
||||
* It is used in priority queue.
|
||||
@ -83,21 +88,27 @@ struct SortCursorImpl
|
||||
SortCursorImpl(
|
||||
const Block & header,
|
||||
const Columns & columns,
|
||||
size_t num_rows,
|
||||
const SortDescription & desc_,
|
||||
size_t order_ = 0,
|
||||
IColumn::Permutation * perm = nullptr)
|
||||
: desc(desc_), sort_columns_size(desc.size()), order(order_), need_collation(desc.size())
|
||||
{
|
||||
reset(columns, header, perm);
|
||||
reset(columns, header, num_rows, perm);
|
||||
}
|
||||
|
||||
bool empty() const { return rows == 0; }
|
||||
|
||||
/// Set the cursor to the beginning of the new block.
|
||||
void reset(const Block & block, IColumn::Permutation * perm = nullptr) { reset(block.getColumns(), block, perm); }
|
||||
void reset(const Block & block, IColumn::Permutation * perm = nullptr)
|
||||
{
|
||||
if (block.getColumns().empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty column list in block");
|
||||
reset(block.getColumns(), block, block.getColumns()[0]->size(), perm);
|
||||
}
|
||||
|
||||
/// Set the cursor to the beginning of the new block.
|
||||
void reset(const Columns & columns, const Block & block, IColumn::Permutation * perm = nullptr)
|
||||
void reset(const Columns & columns, const Block & block, UInt64 num_rows, IColumn::Permutation * perm = nullptr)
|
||||
{
|
||||
all_columns.clear();
|
||||
sort_columns.clear();
|
||||
@ -125,7 +136,7 @@ struct SortCursorImpl
|
||||
}
|
||||
|
||||
pos = 0;
|
||||
rows = all_columns[0]->size();
|
||||
rows = num_rows;
|
||||
permutation = perm;
|
||||
}
|
||||
|
||||
|
||||
@ -33,9 +33,12 @@ struct FillColumnDescription
|
||||
DataTypePtr fill_to_type;
|
||||
Field fill_step; /// Default = +1 or -1 according to direction
|
||||
std::optional<IntervalKind> step_kind;
|
||||
Field fill_staleness; /// Default = Null - should not be considered
|
||||
std::optional<IntervalKind> staleness_kind;
|
||||
|
||||
using StepFunction = std::function<void(Field &)>;
|
||||
using StepFunction = std::function<void(Field &, Int32 jumps_count)>;
|
||||
StepFunction step_func;
|
||||
StepFunction staleness_step_func;
|
||||
};
|
||||
|
||||
/// Description of the sorting rule by one column.
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <DataTypes/DataTypeObject.h>
|
||||
#include <DataTypes/DataTypeObjectDeprecated.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/Serializations/SerializationJSON.h>
|
||||
#include <DataTypes/Serializations/SerializationObjectTypedPath.h>
|
||||
#include <DataTypes/Serializations/SerializationObjectDynamicPath.h>
|
||||
@ -230,6 +233,15 @@ MutableColumnPtr DataTypeObject::createColumn() const
|
||||
return ColumnObject::create(std::move(typed_path_columns), max_dynamic_paths, max_dynamic_types);
|
||||
}
|
||||
|
||||
void DataTypeObject::forEachChild(const ChildCallback & callback) const
|
||||
{
|
||||
for (const auto & [path, type] : typed_paths)
|
||||
{
|
||||
callback(*type);
|
||||
type->forEachChild(callback);
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
@ -522,6 +534,13 @@ static DataTypePtr createObject(const ASTPtr & arguments, const DataTypeObject::
|
||||
return std::make_shared<DataTypeObject>(schema_format, std::move(typed_paths), std::move(paths_to_skip), std::move(path_regexps_to_skip), max_dynamic_paths, max_dynamic_types);
|
||||
}
|
||||
|
||||
const DataTypePtr & DataTypeObject::getTypeOfSharedData()
|
||||
{
|
||||
/// Array(Tuple(String, String))
|
||||
static const DataTypePtr type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeTuple>(DataTypes{std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()}, Names{"paths", "values"}));
|
||||
return type;
|
||||
}
|
||||
|
||||
static DataTypePtr createJSON(const ASTPtr & arguments)
|
||||
{
|
||||
auto context = CurrentThread::getQueryContext();
|
||||
|
||||
@ -50,6 +50,8 @@ public:
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
void forEachChild(const ChildCallback &) const override;
|
||||
|
||||
bool hasDynamicSubcolumnsData() const override { return true; }
|
||||
std::unique_ptr<SubstreamData> getDynamicSubcolumnData(std::string_view subcolumn_name, const SubstreamData & data, bool throw_if_null) const override;
|
||||
|
||||
@ -63,6 +65,9 @@ public:
|
||||
size_t getMaxDynamicTypes() const { return max_dynamic_types; }
|
||||
size_t getMaxDynamicPaths() const { return max_dynamic_paths; }
|
||||
|
||||
/// Shared data has type Array(Tuple(String, String)).
|
||||
static const DataTypePtr & getTypeOfSharedData();
|
||||
|
||||
private:
|
||||
SchemaFormat schema_format;
|
||||
/// Set of paths with types that were specified in type declaration.
|
||||
|
||||
@ -25,7 +25,7 @@ SerializationObject::SerializationObject(
|
||||
: typed_path_serializations(std::move(typed_path_serializations_))
|
||||
, paths_to_skip(paths_to_skip_)
|
||||
, dynamic_serialization(std::make_shared<SerializationDynamic>())
|
||||
, shared_data_serialization(getTypeOfSharedData()->getDefaultSerialization())
|
||||
, shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
|
||||
{
|
||||
/// We will need sorted order of typed paths to serialize them in order for consistency.
|
||||
sorted_typed_paths.reserve(typed_path_serializations.size());
|
||||
@ -38,13 +38,6 @@ SerializationObject::SerializationObject(
|
||||
path_regexps_to_skip.emplace_back(regexp_str);
|
||||
}
|
||||
|
||||
const DataTypePtr & SerializationObject::getTypeOfSharedData()
|
||||
{
|
||||
/// Array(Tuple(String, String))
|
||||
static const DataTypePtr type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeTuple>(DataTypes{std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()}, Names{"paths", "values"}));
|
||||
return type;
|
||||
}
|
||||
|
||||
bool SerializationObject::shouldSkipPath(const String & path) const
|
||||
{
|
||||
if (paths_to_skip.contains(path))
|
||||
@ -168,7 +161,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings,
|
||||
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
auto shared_data_substream_data = SubstreamData(shared_data_serialization)
|
||||
.withType(getTypeOfSharedData())
|
||||
.withType(DataTypeObject::getTypeOfSharedData())
|
||||
.withColumn(column_object ? column_object->getSharedDataPtr() : nullptr)
|
||||
.withSerializationInfo(data.serialization_info)
|
||||
.withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr);
|
||||
|
||||
@ -111,9 +111,6 @@ private:
|
||||
DeserializeBinaryBulkSettings & settings,
|
||||
SubstreamsDeserializeStatesCache * cache);
|
||||
|
||||
/// Shared data has type Array(Tuple(String, String)).
|
||||
static const DataTypePtr & getTypeOfSharedData();
|
||||
|
||||
struct TypedPathSubcolumnCreator : public ISubcolumnCreator
|
||||
{
|
||||
String path;
|
||||
|
||||
@ -18,7 +18,7 @@ SerializationObjectDynamicPath::SerializationObjectDynamicPath(
|
||||
, path(path_)
|
||||
, path_subcolumn(path_subcolumn_)
|
||||
, dynamic_serialization(std::make_shared<SerializationDynamic>())
|
||||
, shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization())
|
||||
, shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
|
||||
, max_dynamic_types(max_dynamic_types_)
|
||||
{
|
||||
}
|
||||
@ -67,8 +67,8 @@ void SerializationObjectDynamicPath::enumerateStreams(
|
||||
{
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
auto shared_data_substream_data = SubstreamData(shared_data_serialization)
|
||||
.withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr)
|
||||
.withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr)
|
||||
.withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr)
|
||||
.withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr)
|
||||
.withSerializationInfo(data.serialization_info)
|
||||
.withDeserializeState(deserialize_state->nested_state);
|
||||
settings.path.back().data = shared_data_substream_data;
|
||||
@ -164,7 +164,7 @@ void SerializationObjectDynamicPath::deserializeBinaryBulkWithMultipleStreams(
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
/// Initialize shared_data column if needed.
|
||||
if (result_column->empty())
|
||||
dynamic_path_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn();
|
||||
dynamic_path_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn();
|
||||
size_t prev_size = result_column->size();
|
||||
shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(dynamic_path_state->shared_data, limit, settings, dynamic_path_state->nested_state, cache);
|
||||
/// If we need to read a subcolumn from Dynamic column, create an empty Dynamic column, fill it and extract subcolumn.
|
||||
|
||||
@ -17,7 +17,7 @@ SerializationSubObject::SerializationSubObject(
|
||||
: path_prefix(path_prefix_)
|
||||
, typed_paths_serializations(typed_paths_serializations_)
|
||||
, dynamic_serialization(std::make_shared<SerializationDynamic>())
|
||||
, shared_data_serialization(SerializationObject::getTypeOfSharedData()->getDefaultSerialization())
|
||||
, shared_data_serialization(DataTypeObject::getTypeOfSharedData()->getDefaultSerialization())
|
||||
{
|
||||
}
|
||||
|
||||
@ -64,8 +64,8 @@ void SerializationSubObject::enumerateStreams(
|
||||
/// We will need to read shared data to find all paths with requested prefix.
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
auto shared_data_substream_data = SubstreamData(shared_data_serialization)
|
||||
.withType(data.type ? SerializationObject::getTypeOfSharedData() : nullptr)
|
||||
.withColumn(data.column ? SerializationObject::getTypeOfSharedData()->createColumn() : nullptr)
|
||||
.withType(data.type ? DataTypeObject::getTypeOfSharedData() : nullptr)
|
||||
.withColumn(data.column ? DataTypeObject::getTypeOfSharedData()->createColumn() : nullptr)
|
||||
.withSerializationInfo(data.serialization_info)
|
||||
.withDeserializeState(deserialize_state ? deserialize_state->shared_data_state : nullptr);
|
||||
settings.path.back().data = shared_data_substream_data;
|
||||
@ -208,7 +208,7 @@ void SerializationSubObject::deserializeBinaryBulkWithMultipleStreams(
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
/// If it's a new object column, reinitialize column for shared data.
|
||||
if (result_column->empty())
|
||||
sub_object_state->shared_data = SerializationObject::getTypeOfSharedData()->createColumn();
|
||||
sub_object_state->shared_data = DataTypeObject::getTypeOfSharedData()->createColumn();
|
||||
size_t prev_size = column_object.size();
|
||||
shared_data_serialization->deserializeBinaryBulkWithMultipleStreams(sub_object_state->shared_data, limit, settings, sub_object_state->shared_data_state, cache);
|
||||
settings.path.pop_back();
|
||||
|
||||
@ -307,6 +307,13 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure(
|
||||
if (!columns.empty())
|
||||
columns_part = fmt::format(" AND attname IN ('{}')", boost::algorithm::join(columns, "','"));
|
||||
|
||||
/// Bypassing the error of the missing column `attgenerated` in the system table `pg_attribute` for PostgreSQL versions below 12.
|
||||
/// This trick involves executing a special query to the DBMS in advance to obtain the correct line with comment /// if column has GENERATED.
|
||||
/// The result of the query will be the name of the column `attgenerated` or an empty string declaration for PostgreSQL version 11 and below.
|
||||
/// This change does not degrade the function's performance but restores support for older versions and fix ERROR: column "attgenerated" does not exist.
|
||||
pqxx::result gen_result{tx.exec("select case when current_setting('server_version_num')::int < 120000 then '''''' else 'attgenerated' end as generated")};
|
||||
std::string generated = gen_result[0][0].as<std::string>();
|
||||
|
||||
std::string query = fmt::format(
|
||||
"SELECT attname AS name, " /// column name
|
||||
"format_type(atttypid, atttypmod) AS type, " /// data type
|
||||
@ -315,11 +322,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure(
|
||||
"atttypid as type_id, "
|
||||
"atttypmod as type_modifier, "
|
||||
"attnum as att_num, "
|
||||
"attgenerated as generated " /// if column has GENERATED
|
||||
"{} as generated " /// if column has GENERATED
|
||||
"FROM pg_attribute "
|
||||
"WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) {}"
|
||||
"AND NOT attisdropped AND attnum > 0 "
|
||||
"ORDER BY attnum ASC", where, columns_part);
|
||||
"ORDER BY attnum ASC", generated, where, columns_part); /// Now we use variable `generated` to form query string. End of trick.
|
||||
|
||||
auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table);
|
||||
table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false);
|
||||
|
||||
@ -32,6 +32,8 @@ void enableAllExperimentalSettings(ContextMutablePtr context)
|
||||
|
||||
context->setSetting("allow_suspicious_low_cardinality_types", 1);
|
||||
context->setSetting("allow_suspicious_fixed_string_types", 1);
|
||||
context->setSetting("allow_suspicious_types_in_group_by", 1);
|
||||
context->setSetting("allow_suspicious_types_in_order_by", 1);
|
||||
context->setSetting("allow_suspicious_indices", 1);
|
||||
context->setSetting("allow_suspicious_codecs", 1);
|
||||
context->setSetting("allow_hyperscan", 1);
|
||||
|
||||
@ -790,6 +790,7 @@ bool CachedOnDiskReadBufferFromFile::writeCache(char * data, size_t size, size_t
|
||||
LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText());
|
||||
return false;
|
||||
}
|
||||
chassert(file_segment.state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
|
||||
throw;
|
||||
}
|
||||
|
||||
|
||||
@ -3921,7 +3921,7 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
WrapperType createTupleToObjectWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const
|
||||
WrapperType createTupleToObjectDeprecatedWrapper(const DataTypeTuple & from_tuple, bool has_nullable_subcolumns) const
|
||||
{
|
||||
if (!from_tuple.haveExplicitNames())
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH,
|
||||
@ -3968,7 +3968,7 @@ private:
|
||||
};
|
||||
}
|
||||
|
||||
WrapperType createMapToObjectWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const
|
||||
WrapperType createMapToObjectDeprecatedWrapper(const DataTypeMap & from_map, bool has_nullable_subcolumns) const
|
||||
{
|
||||
auto key_value_types = from_map.getKeyValueTypes();
|
||||
|
||||
@ -4048,11 +4048,11 @@ private:
|
||||
{
|
||||
if (const auto * from_tuple = checkAndGetDataType<DataTypeTuple>(from_type.get()))
|
||||
{
|
||||
return createTupleToObjectWrapper(*from_tuple, to_type->hasNullableSubcolumns());
|
||||
return createTupleToObjectDeprecatedWrapper(*from_tuple, to_type->hasNullableSubcolumns());
|
||||
}
|
||||
else if (const auto * from_map = checkAndGetDataType<DataTypeMap>(from_type.get()))
|
||||
{
|
||||
return createMapToObjectWrapper(*from_map, to_type->hasNullableSubcolumns());
|
||||
return createMapToObjectDeprecatedWrapper(*from_map, to_type->hasNullableSubcolumns());
|
||||
}
|
||||
else if (checkAndGetDataType<DataTypeString>(from_type.get()))
|
||||
{
|
||||
@ -4081,23 +4081,43 @@ private:
|
||||
"Cast to Object can be performed only from flatten named Tuple, Map or String. Got: {}", from_type->getName());
|
||||
}
|
||||
|
||||
|
||||
WrapperType createObjectWrapper(const DataTypePtr & from_type, const DataTypeObject * to_object) const
|
||||
{
|
||||
if (checkAndGetDataType<DataTypeString>(from_type.get()))
|
||||
{
|
||||
return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count)
|
||||
{
|
||||
auto res = ConvertImplGenericFromString<true>::execute(arguments, result_type, nullable_source, input_rows_count, context)->assumeMutable();
|
||||
res->finalize();
|
||||
return res;
|
||||
return ConvertImplGenericFromString<true>::execute(arguments, result_type, nullable_source, input_rows_count, context);
|
||||
};
|
||||
}
|
||||
|
||||
/// Cast Tuple/Object/Map to JSON type through serializing into JSON string and parsing back into JSON column.
|
||||
/// Potentially we can do smarter conversion Tuple -> JSON with type preservation, but it's questionable how exactly Tuple should be
|
||||
/// converted to JSON (for example, should we recursively convert nested Array(Tuple) to Array(JSON) or not, should we infer types from String fields, etc).
|
||||
if (checkAndGetDataType<DataTypeObjectDeprecated>(from_type.get()) || checkAndGetDataType<DataTypeTuple>(from_type.get()) || checkAndGetDataType<DataTypeMap>(from_type.get()))
|
||||
{
|
||||
return [this](ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count)
|
||||
{
|
||||
auto json_string = ColumnString::create();
|
||||
ColumnStringHelpers::WriteHelper write_helper(assert_cast<ColumnString &>(*json_string), input_rows_count);
|
||||
auto & write_buffer = write_helper.getWriteBuffer();
|
||||
FormatSettings format_settings = context ? getFormatSettings(context) : FormatSettings{};
|
||||
auto serialization = arguments[0].type->getDefaultSerialization();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
serialization->serializeTextJSON(*arguments[0].column, i, write_buffer, format_settings);
|
||||
write_helper.rowWritten();
|
||||
}
|
||||
write_helper.finalize();
|
||||
|
||||
ColumnsWithTypeAndName args_with_json_string = {ColumnWithTypeAndName(json_string->getPtr(), std::make_shared<DataTypeString>(), "")};
|
||||
return ConvertImplGenericFromString<true>::execute(args_with_json_string, result_type, nullable_source, input_rows_count, context);
|
||||
};
|
||||
}
|
||||
|
||||
/// TODO: support CAST between JSON types with different parameters
|
||||
/// support CAST from Map to JSON
|
||||
/// support CAST from Tuple to JSON
|
||||
/// support CAST from Object('json') to JSON
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName());
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Cast to {} can be performed only from String/Map/Object/Tuple. Got: {}", magic_enum::enum_name(to_object->getSchemaFormat()), from_type->getName());
|
||||
}
|
||||
|
||||
WrapperType createVariantToVariantWrapper(const DataTypeVariant & from_variant, const DataTypeVariant & to_variant) const
|
||||
|
||||
@ -24,92 +24,7 @@ namespace ErrorCodes
|
||||
|
||||
void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast)
|
||||
{
|
||||
if (!ast)
|
||||
{
|
||||
chassert(false);
|
||||
return;
|
||||
}
|
||||
|
||||
/// FIXME: this helper should use updatePointerToChild(), but
|
||||
/// forEachPointerToChild() is not implemented for ASTColumnDeclaration
|
||||
/// (and also some members should be adjusted for this).
|
||||
const auto visit_child_with_shared_ptr = [&](ASTPtr & child)
|
||||
{
|
||||
if (!child)
|
||||
return;
|
||||
|
||||
auto * old_value = child.get();
|
||||
visit(child);
|
||||
|
||||
// child did not change
|
||||
if (old_value == child.get())
|
||||
return;
|
||||
|
||||
// child changed, we need to modify it in the list of children of the parent also
|
||||
for (auto & current_child : ast->children)
|
||||
{
|
||||
if (current_child.get() == old_value)
|
||||
current_child = child;
|
||||
}
|
||||
};
|
||||
|
||||
if (auto * col_decl = ast->as<ASTColumnDeclaration>())
|
||||
{
|
||||
visit_child_with_shared_ptr(col_decl->default_expression);
|
||||
visit_child_with_shared_ptr(col_decl->ttl);
|
||||
return;
|
||||
}
|
||||
|
||||
if (auto * storage = ast->as<ASTStorage>())
|
||||
{
|
||||
const auto visit_child = [&](IAST * & child)
|
||||
{
|
||||
if (!child)
|
||||
return;
|
||||
|
||||
if (const auto * function = child->template as<ASTFunction>())
|
||||
{
|
||||
std::unordered_set<std::string> udf_in_replace_process;
|
||||
auto replace_result = tryToReplaceFunction(*function, udf_in_replace_process);
|
||||
if (replace_result)
|
||||
ast->setOrReplace(child, replace_result);
|
||||
}
|
||||
|
||||
visit(child);
|
||||
};
|
||||
|
||||
visit_child(storage->partition_by);
|
||||
visit_child(storage->primary_key);
|
||||
visit_child(storage->order_by);
|
||||
visit_child(storage->sample_by);
|
||||
visit_child(storage->ttl_table);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (auto * alter = ast->as<ASTAlterCommand>())
|
||||
{
|
||||
/// It is OK to use updatePointerToChild() because ASTAlterCommand implements forEachPointerToChild()
|
||||
const auto visit_child_update_parent = [&](ASTPtr & child)
|
||||
{
|
||||
if (!child)
|
||||
return;
|
||||
|
||||
auto * old_ptr = child.get();
|
||||
visit(child);
|
||||
auto * new_ptr = child.get();
|
||||
|
||||
/// Some AST classes have naked pointers to children elements as members.
|
||||
/// We have to replace them if the child was replaced.
|
||||
if (new_ptr != old_ptr)
|
||||
ast->updatePointerToChild(old_ptr, new_ptr);
|
||||
};
|
||||
|
||||
for (auto & children : alter->children)
|
||||
visit_child_update_parent(children);
|
||||
|
||||
return;
|
||||
}
|
||||
chassert(ast);
|
||||
|
||||
if (const auto * function = ast->template as<ASTFunction>())
|
||||
{
|
||||
@ -120,7 +35,19 @@ void UserDefinedSQLFunctionVisitor::visit(ASTPtr & ast)
|
||||
}
|
||||
|
||||
for (auto & child : ast->children)
|
||||
{
|
||||
if (!child)
|
||||
return;
|
||||
|
||||
auto * old_ptr = child.get();
|
||||
visit(child);
|
||||
auto * new_ptr = child.get();
|
||||
|
||||
/// Some AST classes have naked pointers to children elements as members.
|
||||
/// We have to replace them if the child was replaced.
|
||||
if (new_ptr != old_ptr)
|
||||
ast->updatePointerToChild(old_ptr, new_ptr);
|
||||
}
|
||||
}
|
||||
|
||||
void UserDefinedSQLFunctionVisitor::visit(IAST * ast)
|
||||
|
||||
@ -83,7 +83,8 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc
|
||||
queue_iterator->invalidate();
|
||||
|
||||
chassert(candidate->releasable());
|
||||
candidate->file_segment->resetQueueIterator();
|
||||
candidate->file_segment->markDelayedRemovalAndResetQueueIterator();
|
||||
|
||||
/// We need to set removed flag in file segment metadata,
|
||||
/// because in dynamic cache resize we first remove queue entries,
|
||||
/// then evict which also removes file segment metadata,
|
||||
|
||||
@ -172,10 +172,11 @@ void FileSegment::setQueueIterator(Priority::IteratorPtr iterator)
|
||||
queue_iterator = iterator;
|
||||
}
|
||||
|
||||
void FileSegment::resetQueueIterator()
|
||||
void FileSegment::markDelayedRemovalAndResetQueueIterator()
|
||||
{
|
||||
auto lk = lock();
|
||||
queue_iterator.reset();
|
||||
on_delayed_removal = true;
|
||||
queue_iterator = {};
|
||||
}
|
||||
|
||||
size_t FileSegment::getCurrentWriteOffset() const
|
||||
@ -701,6 +702,8 @@ void FileSegment::complete(bool allow_background_download)
|
||||
case State::PARTIALLY_DOWNLOADED:
|
||||
{
|
||||
chassert(current_downloaded_size > 0);
|
||||
chassert(fs::exists(getPath()));
|
||||
chassert(fs::file_size(getPath()) > 0);
|
||||
|
||||
if (is_last_holder)
|
||||
{
|
||||
@ -843,29 +846,60 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock)
|
||||
}
|
||||
}
|
||||
|
||||
if (download_state == State::DOWNLOADED)
|
||||
switch (download_state.load())
|
||||
{
|
||||
chassert(downloader_id.empty());
|
||||
chassert(downloaded_size == reserved_size);
|
||||
chassert(downloaded_size == range().size());
|
||||
chassert(downloaded_size > 0);
|
||||
chassert(std::filesystem::file_size(getPath()) > 0);
|
||||
check_iterator(queue_iterator);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (download_state == State::DOWNLOADING)
|
||||
{
|
||||
chassert(!downloader_id.empty());
|
||||
}
|
||||
else if (download_state == State::PARTIALLY_DOWNLOADED
|
||||
|| download_state == State::EMPTY)
|
||||
case State::EMPTY:
|
||||
{
|
||||
chassert(downloader_id.empty());
|
||||
chassert(!fs::exists(getPath()));
|
||||
chassert(!queue_iterator);
|
||||
break;
|
||||
}
|
||||
case State::DOWNLOADED:
|
||||
{
|
||||
chassert(downloader_id.empty());
|
||||
|
||||
chassert(reserved_size >= downloaded_size);
|
||||
check_iterator(queue_iterator);
|
||||
chassert(downloaded_size == reserved_size);
|
||||
chassert(downloaded_size == range().size());
|
||||
chassert(downloaded_size > 0);
|
||||
chassert(fs::file_size(getPath()) > 0);
|
||||
|
||||
chassert(queue_iterator || on_delayed_removal);
|
||||
check_iterator(queue_iterator);
|
||||
break;
|
||||
}
|
||||
case State::DOWNLOADING:
|
||||
{
|
||||
chassert(!downloader_id.empty());
|
||||
if (downloaded_size)
|
||||
{
|
||||
chassert(queue_iterator);
|
||||
chassert(fs::file_size(getPath()) > 0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case State::PARTIALLY_DOWNLOADED:
|
||||
{
|
||||
chassert(downloader_id.empty());
|
||||
|
||||
chassert(reserved_size >= downloaded_size);
|
||||
chassert(downloaded_size > 0);
|
||||
chassert(fs::file_size(getPath()) > 0);
|
||||
|
||||
chassert(queue_iterator);
|
||||
check_iterator(queue_iterator);
|
||||
break;
|
||||
}
|
||||
case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION:
|
||||
{
|
||||
chassert(reserved_size >= downloaded_size);
|
||||
check_iterator(queue_iterator);
|
||||
break;
|
||||
}
|
||||
case State::DETACHED:
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -993,7 +1027,12 @@ FileSegmentsHolder::FileSegmentsHolder(FileSegments && file_segments_)
|
||||
FileSegmentPtr FileSegmentsHolder::getSingleFileSegment() const
|
||||
{
|
||||
if (file_segments.size() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected single file segment, got: {} in holder {}", file_segments.size(), toString());
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Expected single file segment, got: {} in holder {}",
|
||||
file_segments.size(), toString());
|
||||
}
|
||||
return file_segments.front();
|
||||
}
|
||||
|
||||
@ -1004,12 +1043,21 @@ void FileSegmentsHolder::reset()
|
||||
ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size());
|
||||
for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();)
|
||||
{
|
||||
/// One might think it would have been more correct to do `false` here,
|
||||
/// not to allow background download for file segments that we actually did not start reading.
|
||||
/// But actually we would only do that, if those file segments were already read partially by some other thread/query
|
||||
/// but they were not put to the download queue, because current thread was holding them in Holder.
|
||||
/// So as a culprit, we need to allow to happen what would have happened if we did not exist.
|
||||
file_segment_it = completeAndPopFrontImpl(true);
|
||||
try
|
||||
{
|
||||
/// One might think it would have been more correct to do `false` here,
|
||||
/// not to allow background download for file segments that we actually did not start reading.
|
||||
/// But actually we would only do that, if those file segments were already read partially by some other thread/query
|
||||
/// but they were not put to the download queue, because current thread was holding them in Holder.
|
||||
/// So as a culprit, we need to allow to happen what would have happened if we did not exist.
|
||||
file_segment_it = completeAndPopFrontImpl(true);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
chassert(false);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
file_segments.clear();
|
||||
}
|
||||
|
||||
@ -177,7 +177,7 @@ public:
|
||||
|
||||
void setQueueIterator(Priority::IteratorPtr iterator);
|
||||
|
||||
void resetQueueIterator();
|
||||
void markDelayedRemovalAndResetQueueIterator();
|
||||
|
||||
KeyMetadataPtr tryGetKeyMetadata() const;
|
||||
|
||||
@ -249,12 +249,13 @@ private:
|
||||
|
||||
String tryGetPath() const;
|
||||
|
||||
Key file_key;
|
||||
const Key file_key;
|
||||
Range segment_range;
|
||||
const FileSegmentKind segment_kind;
|
||||
/// Size of the segment is not known until it is downloaded and
|
||||
/// can be bigger than max_file_segment_size.
|
||||
const bool is_unbound = false;
|
||||
/// is_unbound == true for temporary data in cache.
|
||||
const bool is_unbound;
|
||||
const bool background_download_enabled;
|
||||
|
||||
std::atomic<State> download_state;
|
||||
@ -279,6 +280,8 @@ private:
|
||||
std::atomic<size_t> hits_count = 0; /// cache hits.
|
||||
std::atomic<size_t> ref_count = 0; /// Used for getting snapshot state
|
||||
|
||||
bool on_delayed_removal = false;
|
||||
|
||||
CurrentMetrics::Increment metric_increment{CurrentMetrics::CacheFileSegments};
|
||||
};
|
||||
|
||||
|
||||
@ -940,7 +940,16 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl(
|
||||
if (file_segment->queue_iterator && invalidate_queue_entry)
|
||||
file_segment->queue_iterator->invalidate();
|
||||
|
||||
file_segment->detach(segment_lock, *this);
|
||||
try
|
||||
{
|
||||
file_segment->detach(segment_lock, *this);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
chassert(false);
|
||||
/// Do not rethrow, we must delete the file below.
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
@ -990,8 +999,8 @@ void LockedKey::shrinkFileSegmentToDownloadedSize(
|
||||
* because of no space left in cache, we need to be able to cut file segment's size to downloaded_size.
|
||||
*/
|
||||
|
||||
auto metadata = getByOffset(offset);
|
||||
const auto & file_segment = metadata->file_segment;
|
||||
auto file_segment_metadata = getByOffset(offset);
|
||||
const auto & file_segment = file_segment_metadata->file_segment;
|
||||
chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
|
||||
|
||||
const size_t downloaded_size = file_segment->getDownloadedSize();
|
||||
@ -1006,15 +1015,15 @@ void LockedKey::shrinkFileSegmentToDownloadedSize(
|
||||
chassert(file_segment->reserved_size >= downloaded_size);
|
||||
int64_t diff = file_segment->reserved_size - downloaded_size;
|
||||
|
||||
metadata->file_segment = std::make_shared<FileSegment>(
|
||||
file_segment_metadata->file_segment = std::make_shared<FileSegment>(
|
||||
getKey(), offset, downloaded_size, FileSegment::State::DOWNLOADED,
|
||||
CreateFileSegmentSettings(file_segment->getKind()), false,
|
||||
file_segment->cache, key_metadata, file_segment->queue_iterator);
|
||||
|
||||
if (diff)
|
||||
metadata->getQueueIterator()->decrementSize(diff);
|
||||
file_segment_metadata->getQueueIterator()->decrementSize(diff);
|
||||
|
||||
chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
|
||||
chassert(file_segment_metadata->file_segment->assertCorrectnessUnlocked(segment_lock));
|
||||
}
|
||||
|
||||
bool LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &)
|
||||
|
||||
@ -60,17 +60,6 @@ public:
|
||||
IBlocksStreamPtr
|
||||
getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override;
|
||||
|
||||
|
||||
bool isCloneSupported() const override
|
||||
{
|
||||
return !getTotals() && getTotalRowCount() == 0;
|
||||
}
|
||||
|
||||
std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_, const Block &, const Block & right_sample_block_) const override
|
||||
{
|
||||
return std::make_shared<ConcurrentHashJoin>(context, table_join_, slots, right_sample_block_, stats_collecting_params);
|
||||
}
|
||||
|
||||
private:
|
||||
struct InternalHashJoin
|
||||
{
|
||||
|
||||
@ -105,6 +105,8 @@ namespace Setting
|
||||
extern const SettingsBool query_plan_aggregation_in_order;
|
||||
extern const SettingsBool query_plan_read_in_order;
|
||||
extern const SettingsUInt64 use_index_for_in_with_subqueries_max_values;
|
||||
extern const SettingsBool allow_suspicious_types_in_group_by;
|
||||
extern const SettingsBool allow_suspicious_types_in_order_by;
|
||||
}
|
||||
|
||||
|
||||
@ -118,6 +120,7 @@ namespace ErrorCodes
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int UNKNOWN_IDENTIFIER;
|
||||
extern const int UNKNOWN_TYPE_OF_AST_NODE;
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -1368,6 +1371,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join);
|
||||
|
||||
ASTs asts = select_query->groupBy()->children;
|
||||
NameSet group_by_keys;
|
||||
if (select_query->group_by_with_grouping_sets)
|
||||
{
|
||||
for (const auto & ast : asts)
|
||||
@ -1375,6 +1379,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
for (const auto & ast_element : ast->children)
|
||||
{
|
||||
step.addRequiredOutput(ast_element->getColumnName());
|
||||
group_by_keys.insert(ast_element->getColumnName());
|
||||
getRootActions(ast_element, only_types, step.actions()->dag);
|
||||
}
|
||||
}
|
||||
@ -1384,10 +1389,17 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
for (const auto & ast : asts)
|
||||
{
|
||||
step.addRequiredOutput(ast->getColumnName());
|
||||
group_by_keys.insert(ast->getColumnName());
|
||||
getRootActions(ast, only_types, step.actions()->dag);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & result_column : step.getResultColumns())
|
||||
{
|
||||
if (group_by_keys.contains(result_column.name))
|
||||
validateGroupByKeyType(result_column.type);
|
||||
}
|
||||
|
||||
if (optimize_aggregation_in_order)
|
||||
{
|
||||
for (auto & child : asts)
|
||||
@ -1402,6 +1414,26 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
return true;
|
||||
}
|
||||
|
||||
void SelectQueryExpressionAnalyzer::validateGroupByKeyType(const DB::DataTypePtr & key_type) const
|
||||
{
|
||||
if (getContext()->getSettingsRef()[Setting::allow_suspicious_types_in_group_by])
|
||||
return;
|
||||
|
||||
auto check = [](const IDataType & type)
|
||||
{
|
||||
if (isDynamic(type) || isVariant(type))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_COLUMN,
|
||||
"Data types Variant/Dynamic are not allowed in GROUP BY keys, because it can lead to unexpected results. "
|
||||
"Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
|
||||
"its a JSON path subcolumn) or casting this column to a specific data type. "
|
||||
"Set setting allow_suspicious_types_in_group_by = 1 in order to allow it");
|
||||
};
|
||||
|
||||
check(*key_type);
|
||||
key_type->forEachChild(check);
|
||||
}
|
||||
|
||||
void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types)
|
||||
{
|
||||
const auto * select_query = getAggregatingQuery();
|
||||
@ -1599,6 +1631,12 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(
|
||||
with_fill = true;
|
||||
}
|
||||
|
||||
for (const auto & result_column : step.getResultColumns())
|
||||
{
|
||||
if (order_by_keys.contains(result_column.name))
|
||||
validateOrderByKeyType(result_column.type);
|
||||
}
|
||||
|
||||
if (auto interpolate_list = select_query->interpolate())
|
||||
{
|
||||
|
||||
@ -1664,6 +1702,26 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(
|
||||
return actions;
|
||||
}
|
||||
|
||||
void SelectQueryExpressionAnalyzer::validateOrderByKeyType(const DataTypePtr & key_type) const
|
||||
{
|
||||
if (getContext()->getSettingsRef()[Setting::allow_suspicious_types_in_order_by])
|
||||
return;
|
||||
|
||||
auto check = [](const IDataType & type)
|
||||
{
|
||||
if (isDynamic(type) || isVariant(type))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_COLUMN,
|
||||
"Data types Variant/Dynamic are not allowed in ORDER BY keys, because it can lead to unexpected results. "
|
||||
"Consider using a subcolumn with a specific data type instead (for example 'column.Int64' or 'json.some.path.:Int64' if "
|
||||
"its a JSON path subcolumn) or casting this column to a specific data type. "
|
||||
"Set setting allow_suspicious_types_in_order_by = 1 in order to allow it");
|
||||
};
|
||||
|
||||
check(*key_type);
|
||||
key_type->forEachChild(check);
|
||||
}
|
||||
|
||||
bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain, bool only_types)
|
||||
{
|
||||
const auto * select_query = getSelectQuery();
|
||||
@ -1981,7 +2039,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
Block before_prewhere_sample = source_header;
|
||||
if (sanitizeBlock(before_prewhere_sample))
|
||||
{
|
||||
before_prewhere_sample = prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample);
|
||||
ExpressionActions(
|
||||
prewhere_dag_and_flags->dag.clone(),
|
||||
ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
|
||||
auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
|
||||
/// If the filter column is a constant, record it.
|
||||
if (column_elem.column)
|
||||
@ -2013,7 +2073,9 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
before_where_sample = source_header;
|
||||
if (sanitizeBlock(before_where_sample))
|
||||
{
|
||||
before_where_sample = before_where->dag.updateHeader(before_where_sample);
|
||||
ExpressionActions(
|
||||
before_where->dag.clone(),
|
||||
ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
|
||||
|
||||
auto & column_elem
|
||||
= before_where_sample.getByName(query.where()->getColumnName());
|
||||
|
||||
@ -396,6 +396,7 @@ private:
|
||||
ActionsAndProjectInputsFlagPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types);
|
||||
bool appendWhere(ExpressionActionsChain & chain, bool only_types);
|
||||
bool appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, ManyExpressionActions &);
|
||||
void validateGroupByKeyType(const DataTypePtr & key_type) const;
|
||||
void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types);
|
||||
void appendWindowFunctionsArguments(ExpressionActionsChain & chain, bool only_types);
|
||||
|
||||
@ -408,6 +409,7 @@ private:
|
||||
bool appendHaving(ExpressionActionsChain & chain, bool only_types);
|
||||
/// appendSelect
|
||||
ActionsAndProjectInputsFlagPtr appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, ManyExpressionActions &);
|
||||
void validateOrderByKeyType(const DataTypePtr & key_type) const;
|
||||
bool appendLimitBy(ExpressionActionsChain & chain, bool only_types);
|
||||
/// appendProjectResult
|
||||
};
|
||||
|
||||
@ -1,11 +1,24 @@
|
||||
#include <Interpreters/FillingRow.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <cstddef>
|
||||
|
||||
#include <IO/Operators.h>
|
||||
#include <Common/Logger.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <Interpreters/FillingRow.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
constexpr static bool debug_logging_enabled = false;
|
||||
|
||||
template <class... Args>
|
||||
inline static void logDebug(const char * fmt_str, Args&&... args)
|
||||
{
|
||||
if constexpr (debug_logging_enabled)
|
||||
LOG_DEBUG(getLogger("FillingRow"), "{}", fmt::format(fmt::runtime(fmt_str), std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
bool less(const Field & lhs, const Field & rhs, int direction)
|
||||
{
|
||||
if (direction == -1)
|
||||
@ -28,6 +41,10 @@ FillingRow::FillingRow(const SortDescription & sort_description_)
|
||||
: sort_description(sort_description_)
|
||||
{
|
||||
row.resize(sort_description.size());
|
||||
|
||||
constraints.reserve(sort_description.size());
|
||||
for (size_t i = 0; i < size(); ++i)
|
||||
constraints.push_back(getFillDescription(i).fill_to);
|
||||
}
|
||||
|
||||
bool FillingRow::operator<(const FillingRow & other) const
|
||||
@ -63,71 +80,254 @@ bool FillingRow::isNull() const
|
||||
return true;
|
||||
}
|
||||
|
||||
std::pair<bool, bool> FillingRow::next(const FillingRow & to_row)
|
||||
std::optional<Field> FillingRow::doLongJump(const FillColumnDescription & descr, size_t column_ind, const Field & to)
|
||||
{
|
||||
Field shifted_value = row[column_ind];
|
||||
|
||||
if (less(to, shifted_value, getDirection(column_ind)))
|
||||
return std::nullopt;
|
||||
|
||||
for (int32_t step_len = 1, step_no = 0; step_no < 100 && step_len > 0; ++step_no)
|
||||
{
|
||||
Field next_value = shifted_value;
|
||||
descr.step_func(next_value, step_len);
|
||||
|
||||
if (less(to, next_value, getDirection(0)))
|
||||
{
|
||||
step_len /= 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
shifted_value = std::move(next_value);
|
||||
step_len *= 2;
|
||||
}
|
||||
}
|
||||
|
||||
return shifted_value;
|
||||
}
|
||||
|
||||
bool FillingRow::hasSomeConstraints(size_t pos) const
|
||||
{
|
||||
return !constraints[pos].isNull();
|
||||
}
|
||||
|
||||
bool FillingRow::isConstraintsSatisfied(size_t pos) const
|
||||
{
|
||||
chassert(!row[pos].isNull());
|
||||
chassert(hasSomeConstraints(pos));
|
||||
|
||||
int direction = getDirection(pos);
|
||||
logDebug("constraint: {}, row: {}, direction: {}", constraints[pos], row[pos], direction);
|
||||
|
||||
return less(row[pos], constraints[pos], direction);
|
||||
}
|
||||
|
||||
static const Field & findBorder(const Field & constraint, const Field & next_original, int direction)
|
||||
{
|
||||
if (constraint.isNull())
|
||||
return next_original;
|
||||
|
||||
if (next_original.isNull())
|
||||
return constraint;
|
||||
|
||||
if (less(constraint, next_original, direction))
|
||||
return constraint;
|
||||
|
||||
return next_original;
|
||||
}
|
||||
|
||||
bool FillingRow::next(const FillingRow & next_original_row, bool& value_changed)
|
||||
{
|
||||
|
||||
const size_t row_size = size();
|
||||
size_t pos = 0;
|
||||
|
||||
/// Find position we need to increment for generating next row.
|
||||
for (; pos < row_size; ++pos)
|
||||
if (!row[pos].isNull() && !to_row.row[pos].isNull() && !equals(row[pos], to_row.row[pos]))
|
||||
{
|
||||
if (row[pos].isNull())
|
||||
continue;
|
||||
|
||||
const Field & border = findBorder(constraints[pos], next_original_row[pos], getDirection(pos));
|
||||
logDebug("border: {}", border);
|
||||
|
||||
if (!border.isNull() && !equals(row[pos], border))
|
||||
break;
|
||||
}
|
||||
|
||||
if (pos == row_size || less(to_row.row[pos], row[pos], getDirection(pos)))
|
||||
return {false, false};
|
||||
logDebug("pos: {}", pos);
|
||||
|
||||
/// If we have any 'fill_to' value at position greater than 'pos',
|
||||
/// we need to generate rows up to 'fill_to' value.
|
||||
if (pos == row_size)
|
||||
return false;
|
||||
|
||||
if (!next_original_row[pos].isNull() && less(next_original_row[pos], row[pos], getDirection(pos)))
|
||||
return false;
|
||||
|
||||
if (!constraints[pos].isNull() && !less(row[pos], constraints[pos], getDirection(pos)))
|
||||
return false;
|
||||
|
||||
/// If we have any 'fill_to' value at position greater than 'pos' or configured staleness,
|
||||
/// we need to generate rows up to one of this borders.
|
||||
for (size_t i = row_size - 1; i > pos; --i)
|
||||
{
|
||||
auto & fill_column_desc = getFillDescription(i);
|
||||
|
||||
if (fill_column_desc.fill_to.isNull() || row[i].isNull())
|
||||
if (row[i].isNull())
|
||||
continue;
|
||||
|
||||
if (constraints[i].isNull())
|
||||
continue;
|
||||
|
||||
Field next_value = row[i];
|
||||
fill_column_desc.step_func(next_value);
|
||||
if (less(next_value, fill_column_desc.fill_to, getDirection(i)))
|
||||
{
|
||||
row[i] = next_value;
|
||||
initFromDefaults(i + 1);
|
||||
return {true, true};
|
||||
}
|
||||
fill_column_desc.step_func(next_value, 1);
|
||||
|
||||
if (!less(next_value, constraints[i], getDirection(i)))
|
||||
continue;
|
||||
|
||||
row[i] = next_value;
|
||||
initUsingFrom(i + 1);
|
||||
|
||||
value_changed = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
auto next_value = row[pos];
|
||||
getFillDescription(pos).step_func(next_value);
|
||||
getFillDescription(pos).step_func(next_value, 1);
|
||||
|
||||
if (less(to_row.row[pos], next_value, getDirection(pos)) || equals(next_value, getFillDescription(pos).fill_to))
|
||||
return {false, false};
|
||||
if (!next_original_row[pos].isNull() && less(next_original_row[pos], next_value, getDirection(pos)))
|
||||
return false;
|
||||
|
||||
if (!constraints[pos].isNull() && !less(next_value, constraints[pos], getDirection(pos)))
|
||||
return false;
|
||||
|
||||
row[pos] = next_value;
|
||||
if (equals(row[pos], to_row.row[pos]))
|
||||
if (equals(row[pos], next_original_row[pos]))
|
||||
{
|
||||
bool is_less = false;
|
||||
for (size_t i = pos + 1; i < row_size; ++i)
|
||||
{
|
||||
const auto & fill_from = getFillDescription(i).fill_from;
|
||||
if (!fill_from.isNull())
|
||||
row[i] = fill_from;
|
||||
const auto & descr = getFillDescription(i);
|
||||
if (!descr.fill_from.isNull())
|
||||
row[i] = descr.fill_from;
|
||||
else
|
||||
row[i] = to_row.row[i];
|
||||
is_less |= less(row[i], to_row.row[i], getDirection(i));
|
||||
row[i] = next_original_row[i];
|
||||
|
||||
is_less |= (
|
||||
(next_original_row[i].isNull() || less(row[i], next_original_row[i], getDirection(i))) &&
|
||||
(constraints[i].isNull() || less(row[i], constraints[i], getDirection(i)))
|
||||
);
|
||||
}
|
||||
|
||||
return {is_less, true};
|
||||
value_changed = true;
|
||||
return is_less;
|
||||
}
|
||||
|
||||
initFromDefaults(pos + 1);
|
||||
return {true, true};
|
||||
initUsingFrom(pos + 1);
|
||||
|
||||
value_changed = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void FillingRow::initFromDefaults(size_t from_pos)
|
||||
bool FillingRow::shift(const FillingRow & next_original_row, bool& value_changed)
|
||||
{
|
||||
logDebug("next_original_row: {}, current: {}", next_original_row, *this);
|
||||
|
||||
for (size_t pos = 0; pos < size(); ++pos)
|
||||
{
|
||||
if (row[pos].isNull() || next_original_row[pos].isNull() || equals(row[pos], next_original_row[pos]))
|
||||
continue;
|
||||
|
||||
if (less(next_original_row[pos], row[pos], getDirection(pos)))
|
||||
return false;
|
||||
|
||||
std::optional<Field> next_value = doLongJump(getFillDescription(pos), pos, next_original_row[pos]);
|
||||
logDebug("jumped to next value: {}", next_value.value_or("Did not complete"));
|
||||
|
||||
row[pos] = std::move(next_value.value());
|
||||
|
||||
if (equals(row[pos], next_original_row[pos]))
|
||||
{
|
||||
bool is_less = false;
|
||||
for (size_t i = pos + 1; i < size(); ++i)
|
||||
{
|
||||
const auto & descr = getFillDescription(i);
|
||||
if (!descr.fill_from.isNull())
|
||||
row[i] = descr.fill_from;
|
||||
else
|
||||
row[i] = next_original_row[i];
|
||||
|
||||
is_less |= (
|
||||
(next_original_row[i].isNull() || less(row[i], next_original_row[i], getDirection(i))) &&
|
||||
(constraints[i].isNull() || less(row[i], constraints[i], getDirection(i)))
|
||||
);
|
||||
}
|
||||
|
||||
logDebug("is less: {}", is_less);
|
||||
|
||||
value_changed = true;
|
||||
return is_less;
|
||||
}
|
||||
else
|
||||
{
|
||||
initUsingTo(/*from_pos=*/pos + 1);
|
||||
|
||||
value_changed = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FillingRow::hasSomeConstraints() const
|
||||
{
|
||||
for (size_t pos = 0; pos < size(); ++pos)
|
||||
if (hasSomeConstraints(pos))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FillingRow::isConstraintsSatisfied() const
|
||||
{
|
||||
for (size_t pos = 0; pos < size(); ++pos)
|
||||
{
|
||||
if (row[pos].isNull() || !hasSomeConstraints(pos))
|
||||
continue;
|
||||
|
||||
return isConstraintsSatisfied(pos);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void FillingRow::initUsingFrom(size_t from_pos)
|
||||
{
|
||||
for (size_t i = from_pos; i < sort_description.size(); ++i)
|
||||
row[i] = getFillDescription(i).fill_from;
|
||||
}
|
||||
|
||||
void FillingRow::initUsingTo(size_t from_pos)
|
||||
{
|
||||
for (size_t i = from_pos; i < sort_description.size(); ++i)
|
||||
row[i] = getFillDescription(i).fill_to;
|
||||
}
|
||||
|
||||
void FillingRow::updateConstraintsWithStalenessRow(const Columns& base_row, size_t row_ind)
|
||||
{
|
||||
for (size_t i = 0; i < size(); ++i)
|
||||
{
|
||||
const auto& descr = getFillDescription(i);
|
||||
|
||||
if (!descr.fill_staleness.isNull())
|
||||
{
|
||||
Field staleness_border = (*base_row[i])[row_ind];
|
||||
descr.staleness_step_func(staleness_border, 1);
|
||||
constraints[i] = findBorder(descr.fill_to, staleness_border, getDirection(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String FillingRow::dump() const
|
||||
{
|
||||
WriteBufferFromOwnString out;
|
||||
@ -147,3 +347,12 @@ WriteBuffer & operator<<(WriteBuffer & out, const FillingRow & row)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <>
|
||||
struct fmt::formatter<DB::FillingRow> : fmt::formatter<string_view>
|
||||
{
|
||||
constexpr auto format(const DB::FillingRow & row, format_context & ctx) const
|
||||
{
|
||||
return fmt::format_to(ctx.out(), "{}", row.dump());
|
||||
}
|
||||
};
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
#include <Core/SortDescription.h>
|
||||
|
||||
#include <Core/SortDescription.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -15,16 +15,28 @@ bool equals(const Field & lhs, const Field & rhs);
|
||||
*/
|
||||
class FillingRow
|
||||
{
|
||||
/// finds last value <= to
|
||||
std::optional<Field> doLongJump(const FillColumnDescription & descr, size_t column_ind, const Field & to);
|
||||
|
||||
bool hasSomeConstraints(size_t pos) const;
|
||||
bool isConstraintsSatisfied(size_t pos) const;
|
||||
|
||||
public:
|
||||
explicit FillingRow(const SortDescription & sort_description);
|
||||
|
||||
/// Generates next row according to fill 'from', 'to' and 'step' values.
|
||||
/// Return pair of boolean
|
||||
/// apply - true if filling values should be inserted into result set
|
||||
/// value_changed - true if filling row value was changed
|
||||
std::pair<bool, bool> next(const FillingRow & to_row);
|
||||
/// Returns true if filling values should be inserted into result set
|
||||
bool next(const FillingRow & next_original_row, bool& value_changed);
|
||||
|
||||
void initFromDefaults(size_t from_pos = 0);
|
||||
/// Returns true if need to generate some prefix for to_row
|
||||
bool shift(const FillingRow & next_original_row, bool& value_changed);
|
||||
|
||||
bool hasSomeConstraints() const;
|
||||
bool isConstraintsSatisfied() const;
|
||||
|
||||
void initUsingFrom(size_t from_pos = 0);
|
||||
void initUsingTo(size_t from_pos = 0);
|
||||
void updateConstraintsWithStalenessRow(const Columns& base_row, size_t row_ind);
|
||||
|
||||
Field & operator[](size_t index) { return row[index]; }
|
||||
const Field & operator[](size_t index) const { return row[index]; }
|
||||
@ -42,6 +54,7 @@ public:
|
||||
|
||||
private:
|
||||
Row row;
|
||||
Row constraints;
|
||||
SortDescription sort_description;
|
||||
};
|
||||
|
||||
|
||||
@ -36,7 +36,7 @@ public:
|
||||
|
||||
bool isCloneSupported() const override
|
||||
{
|
||||
return !getTotals();
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_,
|
||||
|
||||
@ -383,16 +383,6 @@ size_t HashJoin::getTotalByteCount() const
|
||||
return res;
|
||||
}
|
||||
|
||||
bool HashJoin::isUsedByAnotherAlgorithm() const
|
||||
{
|
||||
return table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) || table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH);
|
||||
}
|
||||
|
||||
bool HashJoin::canRemoveColumnsFromLeftBlock() const
|
||||
{
|
||||
return table_join->enableEnalyzer() && !table_join->hasUsing() && !isUsedByAnotherAlgorithm();
|
||||
}
|
||||
|
||||
void HashJoin::initRightBlockStructure(Block & saved_block_sample)
|
||||
{
|
||||
if (isCrossOrComma(kind))
|
||||
@ -404,7 +394,8 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample)
|
||||
|
||||
bool multiple_disjuncts = !table_join->oneDisjunct();
|
||||
/// We could remove key columns for LEFT | INNER HashJoin but we should keep them for JoinSwitcher (if any).
|
||||
bool save_key_columns = isUsedByAnotherAlgorithm() ||
|
||||
bool save_key_columns = table_join->isEnabledAlgorithm(JoinAlgorithm::AUTO) ||
|
||||
table_join->isEnabledAlgorithm(JoinAlgorithm::GRACE_HASH) ||
|
||||
isRightOrFull(kind) ||
|
||||
multiple_disjuncts ||
|
||||
table_join->getMixedJoinExpression();
|
||||
@ -1237,10 +1228,7 @@ IBlocksStreamPtr HashJoin::getNonJoinedBlocks(const Block & left_sample_block,
|
||||
{
|
||||
if (!JoinCommon::hasNonJoinedBlocks(*table_join))
|
||||
return {};
|
||||
|
||||
size_t left_columns_count = left_sample_block.columns();
|
||||
if (canRemoveColumnsFromLeftBlock())
|
||||
left_columns_count = table_join->getOutputColumns(JoinTableSide::Left).size();
|
||||
|
||||
bool flag_per_row = needUsedFlagsForPerRightTableRow(table_join);
|
||||
if (!flag_per_row)
|
||||
|
||||
@ -127,7 +127,7 @@ public:
|
||||
|
||||
bool isCloneSupported() const override
|
||||
{
|
||||
return !getTotals() && getTotalRowCount() == 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<IJoin> clone(const std::shared_ptr<TableJoin> & table_join_,
|
||||
@ -464,9 +464,6 @@ private:
|
||||
|
||||
bool empty() const;
|
||||
|
||||
bool isUsedByAnotherAlgorithm() const;
|
||||
bool canRemoveColumnsFromLeftBlock() const;
|
||||
|
||||
void validateAdditionalFilterExpression(std::shared_ptr<ExpressionActions> additional_filter_expression);
|
||||
bool needUsedFlagsForPerRightTableRow(std::shared_ptr<TableJoin> table_join_) const;
|
||||
|
||||
|
||||
@ -56,6 +56,7 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
|
||||
const auto & key_names = !is_join_get ? onexprs[i].key_names_left : onexprs[i].key_names_right;
|
||||
join_on_keys.emplace_back(block, key_names, onexprs[i].condColumnNames().first, join.key_sizes[i]);
|
||||
}
|
||||
size_t existing_columns = block.columns();
|
||||
|
||||
/** If you use FULL or RIGHT JOIN, then the columns from the "left" table must be materialized.
|
||||
* Because if they are constants, then in the "not joined" rows, they may have different values
|
||||
@ -98,22 +99,6 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
|
||||
added_columns.buildJoinGetOutput();
|
||||
else
|
||||
added_columns.buildOutput();
|
||||
|
||||
const auto & table_join = join.table_join;
|
||||
std::set<size_t> block_columns_to_erase;
|
||||
if (join.canRemoveColumnsFromLeftBlock())
|
||||
{
|
||||
std::unordered_set<String> left_output_columns;
|
||||
for (const auto & out_column : table_join->getOutputColumns(JoinTableSide::Left))
|
||||
left_output_columns.insert(out_column.name);
|
||||
for (size_t i = 0; i < block.columns(); ++i)
|
||||
{
|
||||
if (!left_output_columns.contains(block.getByPosition(i).name))
|
||||
block_columns_to_erase.insert(i);
|
||||
}
|
||||
}
|
||||
size_t existing_columns = block.columns();
|
||||
|
||||
for (size_t i = 0; i < added_columns.size(); ++i)
|
||||
block.insert(added_columns.moveColumn(i));
|
||||
|
||||
@ -175,7 +160,6 @@ Block HashJoinMethods<KIND, STRICTNESS, MapsTemplate>::joinBlockImpl(
|
||||
block.safeGetByPosition(pos).column = block.safeGetByPosition(pos).column->replicate(*offsets_to_replicate);
|
||||
}
|
||||
}
|
||||
block.erase(block_columns_to_erase);
|
||||
return remaining_block;
|
||||
}
|
||||
|
||||
|
||||
@ -1888,9 +1888,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
expressions.join,
|
||||
settings[Setting::max_block_size],
|
||||
max_streams,
|
||||
/* required_output_ = */ NameSet{},
|
||||
analysis_result.optimize_read_in_order,
|
||||
/* use_new_analyzer_ = */ false);
|
||||
analysis_result.optimize_read_in_order);
|
||||
|
||||
join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType()));
|
||||
std::vector<QueryPlanPtr> plans;
|
||||
|
||||
@ -41,7 +41,6 @@ namespace DB
|
||||
namespace Setting
|
||||
{
|
||||
extern const SettingsBool allow_experimental_join_right_table_sorting;
|
||||
extern const SettingsBool allow_experimental_analyzer;
|
||||
extern const SettingsUInt64 cross_join_min_bytes_to_compress;
|
||||
extern const SettingsUInt64 cross_join_min_rows_to_compress;
|
||||
extern const SettingsUInt64 default_max_bytes_in_join;
|
||||
@ -144,7 +143,6 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary
|
||||
, max_memory_usage(settings[Setting::max_memory_usage])
|
||||
, tmp_volume(tmp_volume_)
|
||||
, tmp_data(tmp_data_)
|
||||
, enable_analyzer(settings[Setting::allow_experimental_analyzer])
|
||||
{
|
||||
}
|
||||
|
||||
@ -163,8 +161,6 @@ void TableJoin::resetCollected()
|
||||
clauses.clear();
|
||||
columns_from_joined_table.clear();
|
||||
columns_added_by_join.clear();
|
||||
columns_from_left_table.clear();
|
||||
result_columns_from_left_table.clear();
|
||||
original_names.clear();
|
||||
renames.clear();
|
||||
left_type_map.clear();
|
||||
@ -207,20 +203,6 @@ size_t TableJoin::rightKeyInclusion(const String & name) const
|
||||
return count;
|
||||
}
|
||||
|
||||
void TableJoin::setInputColumns(NamesAndTypesList left_output_columns, NamesAndTypesList right_output_columns)
|
||||
{
|
||||
columns_from_left_table = std::move(left_output_columns);
|
||||
columns_from_joined_table = std::move(right_output_columns);
|
||||
}
|
||||
|
||||
|
||||
const NamesAndTypesList & TableJoin::getOutputColumns(JoinTableSide side)
|
||||
{
|
||||
if (side == JoinTableSide::Left)
|
||||
return result_columns_from_left_table;
|
||||
return columns_added_by_join;
|
||||
}
|
||||
|
||||
void TableJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix)
|
||||
{
|
||||
NameSet joined_columns;
|
||||
@ -369,18 +351,9 @@ bool TableJoin::rightBecomeNullable(const DataTypePtr & column_type) const
|
||||
return forceNullableRight() && JoinCommon::canBecomeNullable(column_type);
|
||||
}
|
||||
|
||||
void TableJoin::setUsedColumn(const NameAndTypePair & joined_column, JoinTableSide side)
|
||||
{
|
||||
if (side == JoinTableSide::Left)
|
||||
result_columns_from_left_table.push_back(joined_column);
|
||||
else
|
||||
columns_added_by_join.push_back(joined_column);
|
||||
|
||||
}
|
||||
|
||||
void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column)
|
||||
{
|
||||
setUsedColumn(joined_column, JoinTableSide::Right);
|
||||
columns_added_by_join.emplace_back(joined_column);
|
||||
}
|
||||
|
||||
NamesAndTypesList TableJoin::correctedColumnsAddedByJoin() const
|
||||
@ -1022,32 +995,5 @@ size_t TableJoin::getMaxMemoryUsage() const
|
||||
return max_memory_usage;
|
||||
}
|
||||
|
||||
void TableJoin::swapSides()
|
||||
{
|
||||
assertEnableEnalyzer();
|
||||
|
||||
std::swap(key_asts_left, key_asts_right);
|
||||
std::swap(left_type_map, right_type_map);
|
||||
for (auto & clause : clauses)
|
||||
{
|
||||
std::swap(clause.key_names_left, clause.key_names_right);
|
||||
std::swap(clause.on_filter_condition_left, clause.on_filter_condition_right);
|
||||
std::swap(clause.analyzer_left_filter_condition_column_name, clause.analyzer_right_filter_condition_column_name);
|
||||
}
|
||||
|
||||
std::swap(columns_from_left_table, columns_from_joined_table);
|
||||
std::swap(result_columns_from_left_table, columns_added_by_join);
|
||||
|
||||
if (table_join.kind == JoinKind::Left)
|
||||
table_join.kind = JoinKind::Right;
|
||||
else if (table_join.kind == JoinKind::Right)
|
||||
table_join.kind = JoinKind::Left;
|
||||
}
|
||||
|
||||
void TableJoin::assertEnableEnalyzer() const
|
||||
{
|
||||
if (!enable_analyzer)
|
||||
throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "TableJoin: analyzer is disabled");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -167,9 +167,6 @@ private:
|
||||
|
||||
ASOFJoinInequality asof_inequality = ASOFJoinInequality::GreaterOrEquals;
|
||||
|
||||
NamesAndTypesList columns_from_left_table;
|
||||
NamesAndTypesList result_columns_from_left_table;
|
||||
|
||||
/// All columns which can be read from joined table. Duplicating names are qualified.
|
||||
NamesAndTypesList columns_from_joined_table;
|
||||
/// Columns will be added to block by JOIN.
|
||||
@ -205,8 +202,6 @@ private:
|
||||
|
||||
bool is_join_with_constant = false;
|
||||
|
||||
bool enable_analyzer = false;
|
||||
|
||||
Names requiredJoinedNames() const;
|
||||
|
||||
/// Create converting actions and change key column names if required
|
||||
@ -271,8 +266,6 @@ public:
|
||||
VolumePtr getGlobalTemporaryVolume() { return tmp_volume; }
|
||||
|
||||
TemporaryDataOnDiskScopePtr getTempDataOnDisk() { return tmp_data; }
|
||||
bool enableEnalyzer() const { return enable_analyzer; }
|
||||
void assertEnableEnalyzer() const;
|
||||
|
||||
ActionsDAG createJoinedBlockActions(ContextPtr context) const;
|
||||
|
||||
@ -289,7 +282,6 @@ public:
|
||||
}
|
||||
|
||||
bool allowParallelHashJoin() const;
|
||||
void swapSides();
|
||||
|
||||
bool joinUseNulls() const { return join_use_nulls; }
|
||||
|
||||
@ -380,9 +372,6 @@ public:
|
||||
bool leftBecomeNullable(const DataTypePtr & column_type) const;
|
||||
bool rightBecomeNullable(const DataTypePtr & column_type) const;
|
||||
void addJoinedColumn(const NameAndTypePair & joined_column);
|
||||
|
||||
void setUsedColumn(const NameAndTypePair & joined_column, JoinTableSide side);
|
||||
|
||||
void setColumnsAddedByJoin(const NamesAndTypesList & columns_added_by_join_value)
|
||||
{
|
||||
columns_added_by_join = columns_added_by_join_value;
|
||||
@ -408,17 +397,11 @@ public:
|
||||
ASTPtr leftKeysList() const;
|
||||
ASTPtr rightKeysList() const; /// For ON syntax only
|
||||
|
||||
void setColumnsFromJoinedTable(NamesAndTypesList columns_from_joined_table_value, const NameSet & left_table_columns, const String & right_table_prefix, const NamesAndTypesList & columns_from_left_table_)
|
||||
void setColumnsFromJoinedTable(NamesAndTypesList columns_from_joined_table_value, const NameSet & left_table_columns, const String & right_table_prefix)
|
||||
{
|
||||
columns_from_joined_table = std::move(columns_from_joined_table_value);
|
||||
deduplicateAndQualifyColumnNames(left_table_columns, right_table_prefix);
|
||||
result_columns_from_left_table = columns_from_left_table_;
|
||||
columns_from_left_table = columns_from_left_table_;
|
||||
}
|
||||
|
||||
void setInputColumns(NamesAndTypesList left_output_columns, NamesAndTypesList right_output_columns);
|
||||
const NamesAndTypesList & getOutputColumns(JoinTableSide side);
|
||||
|
||||
const NamesAndTypesList & columnsFromJoinedTable() const { return columns_from_joined_table; }
|
||||
const NamesAndTypesList & columnsAddedByJoin() const { return columns_added_by_join; }
|
||||
|
||||
|
||||
@ -1353,15 +1353,12 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
|
||||
|
||||
if (tables_with_columns.size() > 1)
|
||||
{
|
||||
auto columns_from_left_table = tables_with_columns[0].columns;
|
||||
const auto & right_table = tables_with_columns[1];
|
||||
auto columns_from_joined_table = right_table.columns;
|
||||
/// query can use materialized or aliased columns from right joined table,
|
||||
/// we want to request it for right table
|
||||
columns_from_joined_table.insert(columns_from_joined_table.end(), right_table.hidden_columns.begin(), right_table.hidden_columns.end());
|
||||
columns_from_left_table.insert(columns_from_left_table.end(), tables_with_columns[0].hidden_columns.begin(), tables_with_columns[0].hidden_columns.end());
|
||||
result.analyzed_join->setColumnsFromJoinedTable(
|
||||
std::move(columns_from_joined_table), source_columns_set, right_table.table.getQualifiedNamePrefix(), columns_from_left_table);
|
||||
result.analyzed_join->setColumnsFromJoinedTable(std::move(columns_from_joined_table), source_columns_set, right_table.table.getQualifiedNamePrefix());
|
||||
}
|
||||
|
||||
translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns);
|
||||
|
||||
@ -128,4 +128,14 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & format_settings, Fo
|
||||
}
|
||||
}
|
||||
|
||||
void ASTColumnDeclaration::forEachPointerToChild(std::function<void(void **)> f)
|
||||
{
|
||||
f(reinterpret_cast<void **>(&default_expression));
|
||||
f(reinterpret_cast<void **>(&comment));
|
||||
f(reinterpret_cast<void **>(&codec));
|
||||
f(reinterpret_cast<void **>(&statistics_desc));
|
||||
f(reinterpret_cast<void **>(&ttl));
|
||||
f(reinterpret_cast<void **>(&collation));
|
||||
f(reinterpret_cast<void **>(&settings));
|
||||
}
|
||||
}
|
||||
|
||||
@ -29,6 +29,9 @@ public:
|
||||
|
||||
ASTPtr clone() const override;
|
||||
void formatImpl(const FormatSettings & format_settings, FormatState & state, FormatStateStacked frame) const override;
|
||||
|
||||
protected:
|
||||
void forEachPointerToChild(std::function<void(void **)> f) override;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -54,6 +54,11 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " STEP " << (settings.hilite ? hilite_none : "");
|
||||
fill_step->formatImpl(settings, state, frame);
|
||||
}
|
||||
if (auto fill_staleness = getFillStaleness())
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " STALENESS " << (settings.hilite ? hilite_none : "");
|
||||
fill_staleness->formatImpl(settings, state, frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ private:
|
||||
FILL_FROM,
|
||||
FILL_TO,
|
||||
FILL_STEP,
|
||||
FILL_STALENESS,
|
||||
};
|
||||
|
||||
public:
|
||||
@ -32,12 +33,14 @@ public:
|
||||
void setFillFrom(ASTPtr node) { setChild(Child::FILL_FROM, node); }
|
||||
void setFillTo(ASTPtr node) { setChild(Child::FILL_TO, node); }
|
||||
void setFillStep(ASTPtr node) { setChild(Child::FILL_STEP, node); }
|
||||
void setFillStaleness(ASTPtr node) { setChild(Child::FILL_STALENESS, node); }
|
||||
|
||||
/** Collation for locale-specific string comparison. If empty, then sorting done by bytes. */
|
||||
ASTPtr getCollation() const { return getChild(Child::COLLATION); }
|
||||
ASTPtr getFillFrom() const { return getChild(Child::FILL_FROM); }
|
||||
ASTPtr getFillTo() const { return getChild(Child::FILL_TO); }
|
||||
ASTPtr getFillStep() const { return getChild(Child::FILL_STEP); }
|
||||
ASTPtr getFillStaleness() const { return getChild(Child::FILL_STALENESS); }
|
||||
|
||||
String getID(char) const override { return "OrderByElement"; }
|
||||
|
||||
|
||||
@ -546,6 +546,7 @@ namespace DB
|
||||
MR_MACROS(YY, "YY") \
|
||||
MR_MACROS(YYYY, "YYYY") \
|
||||
MR_MACROS(ZKPATH, "ZKPATH") \
|
||||
MR_MACROS(STALENESS, "STALENESS") \
|
||||
|
||||
/// The list of keywords where underscore is intentional
|
||||
#define APPLY_FOR_PARSER_KEYWORDS_WITH_UNDERSCORES(MR_MACROS) \
|
||||
|
||||
@ -31,7 +31,7 @@ CreateQueryUUIDs::CreateQueryUUIDs(const ASTCreateQuery & query, bool generate_r
|
||||
/// If we generate random UUIDs for already existing tables then those UUIDs will not be correct making those inner target table inaccessible.
|
||||
/// Thus it's not safe for example to replace
|
||||
/// "ATTACH MATERIALIZED VIEW mv AS SELECT a FROM b" with
|
||||
/// "ATTACH MATERIALIZED VIEW mv TO INNER UUID '123e4567-e89b-12d3-a456-426614174000' AS SELECT a FROM b"
|
||||
/// "ATTACH MATERIALIZED VIEW mv TO INNER UUID "XXXX" AS SELECT a FROM b"
|
||||
/// This replacement is safe only for CREATE queries when inner target tables don't exist yet.
|
||||
if (!query.attach)
|
||||
{
|
||||
|
||||
@ -2178,6 +2178,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
|
||||
ParserKeyword from(Keyword::FROM);
|
||||
ParserKeyword to(Keyword::TO);
|
||||
ParserKeyword step(Keyword::STEP);
|
||||
ParserKeyword staleness(Keyword::STALENESS);
|
||||
ParserStringLiteral collate_locale_parser;
|
||||
ParserExpressionWithOptionalAlias exp_parser(false);
|
||||
|
||||
@ -2219,6 +2220,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
|
||||
ASTPtr fill_from;
|
||||
ASTPtr fill_to;
|
||||
ASTPtr fill_step;
|
||||
ASTPtr fill_staleness;
|
||||
if (with_fill.ignore(pos, expected))
|
||||
{
|
||||
has_with_fill = true;
|
||||
@ -2230,6 +2232,9 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
|
||||
|
||||
if (step.ignore(pos, expected) && !exp_parser.parse(pos, fill_step, expected))
|
||||
return false;
|
||||
|
||||
if (staleness.ignore(pos, expected) && !exp_parser.parse(pos, fill_staleness, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
auto elem = std::make_shared<ASTOrderByElement>();
|
||||
@ -2244,6 +2249,7 @@ bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
|
||||
elem->setFillFrom(fill_from);
|
||||
elem->setFillTo(fill_to);
|
||||
elem->setFillStep(fill_step);
|
||||
elem->setFillStaleness(fill_staleness);
|
||||
|
||||
node = elem;
|
||||
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/ColumnNode.h>
|
||||
#include <Analyzer/JoinNode.h>
|
||||
|
||||
#include <Planner/PlannerContext.h>
|
||||
|
||||
|
||||
@ -391,6 +391,9 @@ public:
|
||||
|
||||
if (sort_node.hasFillStep())
|
||||
buffer << " STEP " << calculateActionNodeName(sort_node.getFillStep());
|
||||
|
||||
if (sort_node.hasFillStaleness())
|
||||
buffer << " STALENESS " << calculateActionNodeName(sort_node.getFillStaleness());
|
||||
}
|
||||
|
||||
if (i + 1 != order_by_nodes_size)
|
||||
|
||||
@ -104,7 +104,6 @@ namespace Setting
|
||||
extern const SettingsBool optimize_move_to_prewhere;
|
||||
extern const SettingsBool optimize_move_to_prewhere_if_final;
|
||||
extern const SettingsBool use_concurrency_control;
|
||||
extern const SettingsJoinInnerTableSelectionMode query_plan_join_inner_table_selection;
|
||||
}
|
||||
|
||||
namespace ErrorCodes
|
||||
@ -1242,55 +1241,6 @@ void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextP
|
||||
plan_to_add_cast.addStep(std::move(cast_join_columns_step));
|
||||
}
|
||||
|
||||
std::optional<ActionsDAG> createStepToDropColumns(
|
||||
const Block & header,
|
||||
const ColumnIdentifierSet & outer_scope_columns,
|
||||
const PlannerContextPtr & planner_context)
|
||||
{
|
||||
ActionsDAG drop_unused_columns_after_join_actions_dag(header.getColumnsWithTypeAndName());
|
||||
ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
|
||||
std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
|
||||
std::optional<size_t> first_skipped_column_node_index;
|
||||
|
||||
auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag.getOutputs();
|
||||
size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size();
|
||||
|
||||
const auto & global_planner_context = planner_context->getGlobalPlannerContext();
|
||||
|
||||
for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i)
|
||||
{
|
||||
const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i];
|
||||
|
||||
if (drop_unused_columns_after_join_actions_dag_updated_outputs_names.contains(output->result_name)
|
||||
|| !global_planner_context->hasColumnIdentifier(output->result_name))
|
||||
continue;
|
||||
|
||||
if (!outer_scope_columns.contains(output->result_name))
|
||||
{
|
||||
if (!first_skipped_column_node_index)
|
||||
first_skipped_column_node_index = i;
|
||||
continue;
|
||||
}
|
||||
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(output);
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs_names.insert(output->result_name);
|
||||
}
|
||||
|
||||
if (!first_skipped_column_node_index)
|
||||
return {};
|
||||
|
||||
/** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope.
|
||||
*
|
||||
* Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2;
|
||||
*/
|
||||
if (drop_unused_columns_after_join_actions_dag_updated_outputs.empty() && first_skipped_column_node_index)
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]);
|
||||
|
||||
drop_unused_columns_after_join_actions_dag_outputs = std::move(drop_unused_columns_after_join_actions_dag_updated_outputs);
|
||||
|
||||
return drop_unused_columns_after_join_actions_dag;
|
||||
}
|
||||
|
||||
JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_expression,
|
||||
JoinTreeQueryPlan left_join_tree_query_plan,
|
||||
JoinTreeQueryPlan right_join_tree_query_plan,
|
||||
@ -1563,37 +1513,21 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
}
|
||||
|
||||
const Block & left_header = left_plan.getCurrentHeader();
|
||||
auto left_table_names = left_header.getNames();
|
||||
NameSet left_table_names_set(left_table_names.begin(), left_table_names.end());
|
||||
|
||||
auto columns_from_joined_table = right_plan.getCurrentHeader().getNamesAndTypesList();
|
||||
table_join->setColumnsFromJoinedTable(columns_from_joined_table, left_table_names_set, "");
|
||||
|
||||
for (auto & column_from_joined_table : columns_from_joined_table)
|
||||
{
|
||||
/// Add columns from joined table only if they are presented in outer scope, otherwise they can be dropped
|
||||
if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
|
||||
outer_scope_columns.contains(column_from_joined_table.name))
|
||||
table_join->addJoinedColumn(column_from_joined_table);
|
||||
}
|
||||
|
||||
const Block & right_header = right_plan.getCurrentHeader();
|
||||
|
||||
auto columns_from_left_table = left_header.getNamesAndTypesList();
|
||||
auto columns_from_right_table = right_header.getNamesAndTypesList();
|
||||
|
||||
table_join->setInputColumns(columns_from_left_table, columns_from_right_table);
|
||||
|
||||
for (auto & column_from_joined_table : columns_from_left_table)
|
||||
{
|
||||
/// Add columns to output only if they are presented in outer scope, otherwise they can be dropped
|
||||
if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
|
||||
outer_scope_columns.contains(column_from_joined_table.name))
|
||||
table_join->setUsedColumn(column_from_joined_table, JoinTableSide::Left);
|
||||
}
|
||||
|
||||
for (auto & column_from_joined_table : columns_from_right_table)
|
||||
{
|
||||
/// Add columns to output only if they are presented in outer scope, otherwise they can be dropped
|
||||
if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) &&
|
||||
outer_scope_columns.contains(column_from_joined_table.name))
|
||||
table_join->setUsedColumn(column_from_joined_table, JoinTableSide::Right);
|
||||
}
|
||||
|
||||
if (table_join->getOutputColumns(JoinTableSide::Left).empty() && table_join->getOutputColumns(JoinTableSide::Right).empty())
|
||||
{
|
||||
if (!columns_from_left_table.empty())
|
||||
table_join->setUsedColumn(columns_from_left_table.front(), JoinTableSide::Left);
|
||||
else if (!columns_from_right_table.empty())
|
||||
table_join->setUsedColumn(columns_from_right_table.front(), JoinTableSide::Right);
|
||||
}
|
||||
|
||||
auto join_algorithm = chooseJoinAlgorithm(table_join, join_node.getRightTableExpression(), left_header, right_header, planner_context);
|
||||
|
||||
auto result_plan = QueryPlan();
|
||||
@ -1681,26 +1615,13 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
}
|
||||
|
||||
auto join_pipeline_type = join_algorithm->pipelineType();
|
||||
|
||||
ColumnIdentifierSet outer_scope_columns_nonempty;
|
||||
if (outer_scope_columns.empty())
|
||||
{
|
||||
if (left_header.columns() > 1)
|
||||
outer_scope_columns_nonempty.insert(left_header.getByPosition(0).name);
|
||||
else if (right_header.columns() > 1)
|
||||
outer_scope_columns_nonempty.insert(right_header.getByPosition(0).name);
|
||||
}
|
||||
|
||||
auto join_step = std::make_unique<JoinStep>(
|
||||
left_plan.getCurrentHeader(),
|
||||
right_plan.getCurrentHeader(),
|
||||
std::move(join_algorithm),
|
||||
settings[Setting::max_block_size],
|
||||
settings[Setting::max_threads],
|
||||
outer_scope_columns.empty() ? outer_scope_columns_nonempty : outer_scope_columns,
|
||||
false /*optimize_read_in_order*/,
|
||||
true /*optimize_skip_unused_shards*/);
|
||||
join_step->inner_table_selection_mode = settings[Setting::query_plan_join_inner_table_selection];
|
||||
false /*optimize_read_in_order*/);
|
||||
|
||||
join_step->setStepDescription(fmt::format("JOIN {}", join_pipeline_type));
|
||||
|
||||
@ -1711,18 +1632,47 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
result_plan.unitePlans(std::move(join_step), {std::move(plans)});
|
||||
}
|
||||
|
||||
const auto & header_after_join = result_plan.getCurrentHeader();
|
||||
if (header_after_join.columns() > outer_scope_columns.size())
|
||||
ActionsDAG drop_unused_columns_after_join_actions_dag(result_plan.getCurrentHeader().getColumnsWithTypeAndName());
|
||||
ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
|
||||
std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
|
||||
std::optional<size_t> first_skipped_column_node_index;
|
||||
|
||||
auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag.getOutputs();
|
||||
size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size();
|
||||
|
||||
for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i)
|
||||
{
|
||||
auto drop_unused_columns_after_join_actions_dag = createStepToDropColumns(header_after_join, outer_scope_columns, planner_context);
|
||||
if (drop_unused_columns_after_join_actions_dag)
|
||||
const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i];
|
||||
|
||||
const auto & global_planner_context = planner_context->getGlobalPlannerContext();
|
||||
if (drop_unused_columns_after_join_actions_dag_updated_outputs_names.contains(output->result_name)
|
||||
|| !global_planner_context->hasColumnIdentifier(output->result_name))
|
||||
continue;
|
||||
|
||||
if (!outer_scope_columns.contains(output->result_name))
|
||||
{
|
||||
auto drop_unused_columns_after_join_transform_step = std::make_unique<ExpressionStep>(result_plan.getCurrentHeader(), std::move(*drop_unused_columns_after_join_actions_dag));
|
||||
drop_unused_columns_after_join_transform_step->setStepDescription("Drop unused columns after JOIN");
|
||||
result_plan.addStep(std::move(drop_unused_columns_after_join_transform_step));
|
||||
if (!first_skipped_column_node_index)
|
||||
first_skipped_column_node_index = i;
|
||||
continue;
|
||||
}
|
||||
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(output);
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs_names.insert(output->result_name);
|
||||
}
|
||||
|
||||
/** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope.
|
||||
*
|
||||
* Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2;
|
||||
*/
|
||||
if (drop_unused_columns_after_join_actions_dag_updated_outputs.empty() && first_skipped_column_node_index)
|
||||
drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]);
|
||||
|
||||
drop_unused_columns_after_join_actions_dag_outputs = std::move(drop_unused_columns_after_join_actions_dag_updated_outputs);
|
||||
|
||||
auto drop_unused_columns_after_join_transform_step = std::make_unique<ExpressionStep>(result_plan.getCurrentHeader(), std::move(drop_unused_columns_after_join_actions_dag));
|
||||
drop_unused_columns_after_join_transform_step->setStepDescription("DROP unused columns after JOIN");
|
||||
result_plan.addStep(std::move(drop_unused_columns_after_join_transform_step));
|
||||
|
||||
for (const auto & right_join_tree_query_plan_row_policy : right_join_tree_query_plan.used_row_policies)
|
||||
left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ std::pair<Field, DataTypePtr> extractWithFillValue(const QueryTreeNodePtr & node
|
||||
return result;
|
||||
}
|
||||
|
||||
std::pair<Field, std::optional<IntervalKind>> extractWithFillStepValue(const QueryTreeNodePtr & node)
|
||||
std::pair<Field, std::optional<IntervalKind>> extractWithFillValueWithIntervalKind(const QueryTreeNodePtr & node)
|
||||
{
|
||||
const auto & constant_node = node->as<ConstantNode &>();
|
||||
|
||||
@ -77,7 +77,7 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node)
|
||||
|
||||
if (sort_node.hasFillStep())
|
||||
{
|
||||
auto extract_result = extractWithFillStepValue(sort_node.getFillStep());
|
||||
auto extract_result = extractWithFillValueWithIntervalKind(sort_node.getFillStep());
|
||||
fill_column_description.fill_step = std::move(extract_result.first);
|
||||
fill_column_description.step_kind = std::move(extract_result.second);
|
||||
}
|
||||
@ -87,16 +87,36 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node)
|
||||
fill_column_description.fill_step = Field(direction_value);
|
||||
}
|
||||
|
||||
if (sort_node.getFillStaleness())
|
||||
{
|
||||
auto extract_result = extractWithFillValueWithIntervalKind(sort_node.getFillStaleness());
|
||||
fill_column_description.fill_staleness = std::move(extract_result.first);
|
||||
fill_column_description.staleness_kind = std::move(extract_result.second);
|
||||
}
|
||||
|
||||
///////////////////////////////////
|
||||
|
||||
if (applyVisitor(FieldVisitorAccurateEquals(), fill_column_description.fill_step, Field{0}))
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STEP value cannot be zero");
|
||||
|
||||
if (sort_node.hasFillStaleness())
|
||||
{
|
||||
if (sort_node.hasFillFrom())
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STALENESS cannot be used together with WITH FILL FROM");
|
||||
}
|
||||
|
||||
if (sort_node.getSortDirection() == SortDirection::ASCENDING)
|
||||
{
|
||||
if (applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_step, Field{0}))
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STEP value cannot be negative for sorting in ascending direction");
|
||||
|
||||
if (applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_staleness, Field{0}))
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STALENESS value cannot be negative for sorting in ascending direction");
|
||||
|
||||
if (!fill_column_description.fill_from.isNull() && !fill_column_description.fill_to.isNull() &&
|
||||
applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_to, fill_column_description.fill_from))
|
||||
{
|
||||
@ -110,6 +130,10 @@ FillColumnDescription extractWithFillDescription(const SortNode & sort_node)
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STEP value cannot be positive for sorting in descending direction");
|
||||
|
||||
if (applyVisitor(FieldVisitorAccurateLess(), Field{0}, fill_column_description.fill_staleness))
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"WITH FILL STALENESS value cannot be positive for sorting in descending direction");
|
||||
|
||||
if (!fill_column_description.fill_from.isNull() && !fill_column_description.fill_to.isNull() &&
|
||||
applyVisitor(FieldVisitorAccurateLess(), fill_column_description.fill_from, fill_column_description.fill_to))
|
||||
{
|
||||
|
||||
@ -92,8 +92,9 @@ static NamesAndTypesList getHeaderForParquetMetadata()
|
||||
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>()),
|
||||
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
|
||||
Names{"num_values", "null_count", "distinct_count", "min", "max"}),
|
||||
DataTypeFactory::instance().get("Bool"),
|
||||
},
|
||||
Names{"name", "path", "total_compressed_size", "total_uncompressed_size", "have_statistics", "statistics"}))},
|
||||
Names{"name", "path", "total_compressed_size", "total_uncompressed_size", "have_statistics", "statistics", "have_bloom_filter"}))},
|
||||
Names{"num_columns", "num_rows", "total_uncompressed_size", "total_compressed_size", "columns"}))},
|
||||
};
|
||||
return names_and_types;
|
||||
@ -350,6 +351,8 @@ void ParquetMetadataInputFormat::fillColumnChunksMetadata(const std::unique_ptr<
|
||||
fillColumnStatistics(column_chunk_metadata->statistics(), tuple_column.getColumn(5), row_group_metadata->schema()->Column(column_i)->type_length());
|
||||
else
|
||||
tuple_column.getColumn(5).insertDefault();
|
||||
bool have_bloom_filter = column_chunk_metadata->bloom_filter_offset().has_value();
|
||||
assert_cast<ColumnUInt8 &>(tuple_column.getColumn(6)).insertValue(have_bloom_filter);
|
||||
}
|
||||
array_column.getOffsets().push_back(tuple_column.size());
|
||||
}
|
||||
|
||||
@ -24,7 +24,12 @@ void IMergingAlgorithmWithDelayedChunk::initializeQueue(Inputs inputs)
|
||||
continue;
|
||||
|
||||
cursors[source_num] = SortCursorImpl(
|
||||
header, current_inputs[source_num].chunk.getColumns(), description, source_num, current_inputs[source_num].permutation);
|
||||
header,
|
||||
current_inputs[source_num].chunk.getColumns(),
|
||||
current_inputs[source_num].chunk.getNumRows(),
|
||||
description,
|
||||
source_num,
|
||||
current_inputs[source_num].permutation);
|
||||
|
||||
inputs_origin_merge_tree_part_level[source_num] = getPartLevelFromChunk(current_inputs[source_num].chunk);
|
||||
}
|
||||
@ -41,7 +46,7 @@ void IMergingAlgorithmWithDelayedChunk::updateCursor(Input & input, size_t sourc
|
||||
last_chunk_sort_columns = std::move(cursors[source_num].sort_columns);
|
||||
|
||||
current_input.swap(input);
|
||||
cursors[source_num].reset(current_input.chunk.getColumns(), header, current_input.permutation);
|
||||
cursors[source_num].reset(current_input.chunk.getColumns(), header, current_input.chunk.getNumRows(), current_input.permutation);
|
||||
|
||||
inputs_origin_merge_tree_part_level[source_num] = getPartLevelFromChunk(current_input.chunk);
|
||||
|
||||
|
||||
@ -31,7 +31,8 @@ void IMergingAlgorithmWithSharedChunks::initialize(Inputs inputs)
|
||||
|
||||
source.skip_last_row = inputs[source_num].skip_last_row;
|
||||
source.chunk = chunk_allocator.alloc(inputs[source_num].chunk);
|
||||
cursors[source_num] = SortCursorImpl(header, source.chunk->getColumns(), description, source_num, inputs[source_num].permutation);
|
||||
cursors[source_num] = SortCursorImpl(
|
||||
header, source.chunk->getColumns(), source.chunk->getNumRows(), description, source_num, inputs[source_num].permutation);
|
||||
|
||||
source.chunk->all_columns = cursors[source_num].all_columns;
|
||||
source.chunk->sort_columns = cursors[source_num].sort_columns;
|
||||
@ -49,7 +50,7 @@ void IMergingAlgorithmWithSharedChunks::consume(Input & input, size_t source_num
|
||||
auto & source = sources[source_num];
|
||||
source.skip_last_row = input.skip_last_row;
|
||||
source.chunk = chunk_allocator.alloc(input.chunk);
|
||||
cursors[source_num].reset(source.chunk->getColumns(), header, input.permutation);
|
||||
cursors[source_num].reset(source.chunk->getColumns(), header, source.chunk->getNumRows(), input.permutation);
|
||||
|
||||
source.chunk->all_columns = cursors[source_num].all_columns;
|
||||
source.chunk->sort_columns = cursors[source_num].sort_columns;
|
||||
|
||||
@ -59,7 +59,7 @@ void MergingSortedAlgorithm::initialize(Inputs inputs)
|
||||
if (!chunk)
|
||||
continue;
|
||||
|
||||
cursors[source_num] = SortCursorImpl(header, chunk.getColumns(), description, source_num);
|
||||
cursors[source_num] = SortCursorImpl(header, chunk.getColumns(), chunk.getNumRows(), description, source_num);
|
||||
}
|
||||
|
||||
if (sorting_queue_strategy == SortingQueueStrategy::Default)
|
||||
@ -84,7 +84,7 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num)
|
||||
{
|
||||
removeConstAndSparse(input);
|
||||
current_inputs[source_num].swap(input);
|
||||
cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), header);
|
||||
cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), header, current_inputs[source_num].chunk.getNumRows());
|
||||
|
||||
if (sorting_queue_strategy == SortingQueueStrategy::Default)
|
||||
{
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <IO/Operators.h>
|
||||
#include <Common/JSONBuilder.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Processors/Transforms/ColumnPermuteTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -37,37 +36,6 @@ std::vector<std::pair<String, String>> describeJoinActions(const JoinPtr & join)
|
||||
return description;
|
||||
}
|
||||
|
||||
std::vector<size_t> getPermutationForBlock(
|
||||
const Block & block,
|
||||
const Block & lhs_block,
|
||||
const Block & rhs_block,
|
||||
const NameSet & name_filter)
|
||||
{
|
||||
std::vector<size_t> permutation;
|
||||
permutation.reserve(block.columns());
|
||||
Block::NameMap name_map = block.getNamesToIndexesMap();
|
||||
|
||||
bool is_trivial = true;
|
||||
for (const auto & other_block : {lhs_block, rhs_block})
|
||||
{
|
||||
for (const auto & col : other_block)
|
||||
{
|
||||
if (!name_filter.contains(col.name))
|
||||
continue;
|
||||
if (auto it = name_map.find(col.name); it != name_map.end())
|
||||
{
|
||||
is_trivial = is_trivial && it->second == permutation.size();
|
||||
permutation.push_back(it->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (is_trivial && permutation.size() == block.columns())
|
||||
return {};
|
||||
|
||||
return permutation;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
JoinStep::JoinStep(
|
||||
@ -76,15 +44,8 @@ JoinStep::JoinStep(
|
||||
JoinPtr join_,
|
||||
size_t max_block_size_,
|
||||
size_t max_streams_,
|
||||
NameSet required_output_,
|
||||
bool keep_left_read_in_order_,
|
||||
bool use_new_analyzer_)
|
||||
: join(std::move(join_))
|
||||
, max_block_size(max_block_size_)
|
||||
, max_streams(max_streams_)
|
||||
, required_output(std::move(required_output_))
|
||||
, keep_left_read_in_order(keep_left_read_in_order_)
|
||||
, use_new_analyzer(use_new_analyzer_)
|
||||
bool keep_left_read_in_order_)
|
||||
: join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_)
|
||||
{
|
||||
updateInputHeaders({left_header_, right_header_});
|
||||
}
|
||||
@ -94,43 +55,23 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines
|
||||
if (pipelines.size() != 2)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "JoinStep expect two input steps");
|
||||
|
||||
Block lhs_header = pipelines[0]->getHeader();
|
||||
Block rhs_header = pipelines[1]->getHeader();
|
||||
|
||||
if (swap_streams)
|
||||
std::swap(pipelines[0], pipelines[1]);
|
||||
|
||||
if (join->pipelineType() == JoinPipelineType::YShaped)
|
||||
{
|
||||
auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped(
|
||||
std::move(pipelines[0]), std::move(pipelines[1]), join, join_algorithm_header, max_block_size, &processors);
|
||||
std::move(pipelines[0]), std::move(pipelines[1]), join, *output_header, max_block_size, &processors);
|
||||
joined_pipeline->resize(max_streams);
|
||||
return joined_pipeline;
|
||||
}
|
||||
|
||||
auto pipeline = QueryPipelineBuilder::joinPipelinesRightLeft(
|
||||
return QueryPipelineBuilder::joinPipelinesRightLeft(
|
||||
std::move(pipelines[0]),
|
||||
std::move(pipelines[1]),
|
||||
join,
|
||||
join_algorithm_header,
|
||||
*output_header,
|
||||
max_block_size,
|
||||
max_streams,
|
||||
keep_left_read_in_order,
|
||||
&processors);
|
||||
|
||||
if (!use_new_analyzer)
|
||||
return pipeline;
|
||||
|
||||
auto column_permutation = getPermutationForBlock(pipeline->getHeader(), lhs_header, rhs_header, required_output);
|
||||
if (!column_permutation.empty())
|
||||
{
|
||||
pipeline->addSimpleTransform([&column_permutation](const Block & header)
|
||||
{
|
||||
return std::make_shared<ColumnPermuteTransform>(header, column_permutation);
|
||||
});
|
||||
}
|
||||
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
bool JoinStep::allowPushDownToRight() const
|
||||
@ -149,49 +90,17 @@ void JoinStep::describeActions(FormatSettings & settings) const
|
||||
|
||||
for (const auto & [name, value] : describeJoinActions(join))
|
||||
settings.out << prefix << name << ": " << value << '\n';
|
||||
if (swap_streams)
|
||||
settings.out << prefix << "Swapped: true\n";
|
||||
}
|
||||
|
||||
void JoinStep::describeActions(JSONBuilder::JSONMap & map) const
|
||||
{
|
||||
for (const auto & [name, value] : describeJoinActions(join))
|
||||
map.add(name, value);
|
||||
if (swap_streams)
|
||||
map.add("Swapped", true);
|
||||
}
|
||||
|
||||
void JoinStep::setJoin(JoinPtr join_, bool swap_streams_)
|
||||
{
|
||||
join_algorithm_header.clear();
|
||||
swap_streams = swap_streams_;
|
||||
join = std::move(join_);
|
||||
updateOutputHeader();
|
||||
}
|
||||
|
||||
void JoinStep::updateOutputHeader()
|
||||
{
|
||||
if (join_algorithm_header)
|
||||
return;
|
||||
|
||||
const auto & header = swap_streams ? input_headers[1] : input_headers[0];
|
||||
|
||||
Block result_header = JoiningTransform::transformHeader(header, join);
|
||||
join_algorithm_header = result_header;
|
||||
|
||||
if (!use_new_analyzer)
|
||||
{
|
||||
if (swap_streams)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot swap streams without new analyzer");
|
||||
output_header = result_header;
|
||||
return;
|
||||
}
|
||||
|
||||
auto column_permutation = getPermutationForBlock(result_header, input_headers[0], input_headers[1], required_output);
|
||||
if (!column_permutation.empty())
|
||||
result_header = ColumnPermuteTransform::permute(result_header, column_permutation);
|
||||
|
||||
output_header = result_header;
|
||||
output_header = JoiningTransform::transformHeader(input_headers.front(), join);
|
||||
}
|
||||
|
||||
static ITransformingStep::Traits getStorageJoinTraits()
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
|
||||
#include <Processors/QueryPlan/IQueryPlanStep.h>
|
||||
#include <Processors/QueryPlan/ITransformingStep.h>
|
||||
#include <Core/Joins.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -20,9 +19,7 @@ public:
|
||||
JoinPtr join_,
|
||||
size_t max_block_size_,
|
||||
size_t max_streams_,
|
||||
NameSet required_output_,
|
||||
bool keep_left_read_in_order_,
|
||||
bool use_new_analyzer_);
|
||||
bool keep_left_read_in_order_);
|
||||
|
||||
String getName() const override { return "Join"; }
|
||||
|
||||
@ -34,26 +31,16 @@ public:
|
||||
void describeActions(FormatSettings & settings) const override;
|
||||
|
||||
const JoinPtr & getJoin() const { return join; }
|
||||
void setJoin(JoinPtr join_, bool swap_streams_ = false);
|
||||
void setJoin(JoinPtr join_) { join = std::move(join_); }
|
||||
bool allowPushDownToRight() const;
|
||||
|
||||
JoinInnerTableSelectionMode inner_table_selection_mode = JoinInnerTableSelectionMode::Right;
|
||||
|
||||
private:
|
||||
void updateOutputHeader() override;
|
||||
|
||||
/// Header that expected to be returned from IJoin
|
||||
Block join_algorithm_header;
|
||||
|
||||
JoinPtr join;
|
||||
size_t max_block_size;
|
||||
size_t max_streams;
|
||||
|
||||
const NameSet required_output;
|
||||
std::set<size_t> columns_to_remove;
|
||||
bool keep_left_read_in_order;
|
||||
bool use_new_analyzer = false;
|
||||
bool swap_streams = false;
|
||||
};
|
||||
|
||||
/// Special step for the case when Join is already filled.
|
||||
|
||||
@ -113,7 +113,6 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack);
|
||||
void optimizePrewhere(Stack & stack, QueryPlan::Nodes & nodes);
|
||||
void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes);
|
||||
void optimizeAggregationInOrder(QueryPlan::Node & node, QueryPlan::Nodes &);
|
||||
void optimizeJoin(QueryPlan::Node & node, QueryPlan::Nodes &);
|
||||
void optimizeDistinctInOrder(QueryPlan::Node & node, QueryPlan::Nodes &);
|
||||
|
||||
/// A separate tree traverse to apply sorting properties after *InOrder optimizations.
|
||||
|
||||
@ -1,102 +0,0 @@
|
||||
#include <Processors/QueryPlan/ExpressionStep.h>
|
||||
#include <Processors/QueryPlan/FilterStep.h>
|
||||
#include <Processors/QueryPlan/ITransformingStep.h>
|
||||
#include <Processors/QueryPlan/JoinStep.h>
|
||||
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
|
||||
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
|
||||
#include <Processors/QueryPlan/ReadFromMergeTree.h>
|
||||
#include <Processors/QueryPlan/SortingStep.h>
|
||||
#include <Storages/StorageMemory.h>
|
||||
#include <Processors/QueryPlan/ReadFromMemoryStorageStep.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <Interpreters/IJoin.h>
|
||||
#include <Interpreters/HashJoin/HashJoin.h>
|
||||
|
||||
#include <Interpreters/TableJoin.h>
|
||||
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Core/Joins.h>
|
||||
#include <ranges>
|
||||
|
||||
namespace DB::QueryPlanOptimizations
|
||||
{
|
||||
|
||||
static std::optional<UInt64> estimateReadRowsCount(QueryPlan::Node & node)
|
||||
{
|
||||
IQueryPlanStep * step = node.step.get();
|
||||
if (const auto * reading = typeid_cast<const ReadFromMergeTree *>(step))
|
||||
{
|
||||
if (auto analyzed_result = reading->getAnalyzedResult())
|
||||
return analyzed_result->selected_rows;
|
||||
if (auto analyzed_result = reading->selectRangesToRead())
|
||||
return analyzed_result->selected_rows;
|
||||
return {};
|
||||
}
|
||||
|
||||
if (const auto * reading = typeid_cast<const ReadFromMemoryStorageStep *>(step))
|
||||
return reading->getStorage()->totalRows(Settings{});
|
||||
|
||||
if (node.children.size() != 1)
|
||||
return {};
|
||||
|
||||
if (typeid_cast<ExpressionStep *>(step) || typeid_cast<FilterStep *>(step))
|
||||
return estimateReadRowsCount(*node.children.front());
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
void optimizeJoin(QueryPlan::Node & node, QueryPlan::Nodes &)
|
||||
{
|
||||
auto * join_step = typeid_cast<JoinStep *>(node.step.get());
|
||||
if (!join_step || node.children.size() != 2)
|
||||
return;
|
||||
|
||||
const auto & join = join_step->getJoin();
|
||||
if (join->pipelineType() != JoinPipelineType::FillRightFirst || !join->isCloneSupported())
|
||||
return;
|
||||
|
||||
const auto & table_join = join->getTableJoin();
|
||||
|
||||
/// Algorithms other than HashJoin may not support OUTER JOINs
|
||||
if (table_join.kind() != JoinKind::Inner && !typeid_cast<const HashJoin *>(join.get()))
|
||||
return;
|
||||
|
||||
/// fixme: USING clause handled specially in join algorithm, so swap breaks it
|
||||
/// fixme: Swapping for SEMI and ANTI joins should be alright, need to try to enable it and test
|
||||
if (table_join.hasUsing() || table_join.strictness() != JoinStrictness::All)
|
||||
return;
|
||||
|
||||
bool need_swap = false;
|
||||
if (join_step->inner_table_selection_mode == JoinInnerTableSelectionMode::Auto)
|
||||
{
|
||||
auto lhs_extimation = estimateReadRowsCount(*node.children[0]);
|
||||
auto rhs_extimation = estimateReadRowsCount(*node.children[1]);
|
||||
LOG_TRACE(getLogger("optimizeJoin"), "Left table estimation: {}, right table estimation: {}",
|
||||
lhs_extimation.transform(toString<UInt64>).value_or("unknown"),
|
||||
rhs_extimation.transform(toString<UInt64>).value_or("unknown"));
|
||||
|
||||
if (lhs_extimation && rhs_extimation && *lhs_extimation < *rhs_extimation)
|
||||
need_swap = true;
|
||||
}
|
||||
else if (join_step->inner_table_selection_mode == JoinInnerTableSelectionMode::Left)
|
||||
{
|
||||
need_swap = true;
|
||||
}
|
||||
|
||||
if (!need_swap)
|
||||
return;
|
||||
|
||||
const auto & headers = join_step->getInputHeaders();
|
||||
if (headers.size() != 2)
|
||||
return;
|
||||
|
||||
const auto & left_stream_input_header = headers.front();
|
||||
const auto & right_stream_input_header = headers.back();
|
||||
|
||||
auto updated_table_join = std::make_shared<TableJoin>(table_join);
|
||||
updated_table_join->swapSides();
|
||||
auto updated_join = join->clone(updated_table_join, right_stream_input_header, left_stream_input_header);
|
||||
join_step->setJoin(std::move(updated_join), /* swap_streams= */ true);
|
||||
}
|
||||
|
||||
}
|
||||
@ -227,9 +227,6 @@ void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::No
|
||||
/// NOTE: frame cannot be safely used after stack was modified.
|
||||
auto & frame = stack.back();
|
||||
|
||||
if (frame.next_child == 0)
|
||||
optimizeJoin(*frame.node, nodes);
|
||||
|
||||
/// Traverse all children first.
|
||||
if (frame.next_child < frame.node->children.size())
|
||||
{
|
||||
|
||||
@ -35,8 +35,6 @@ public:
|
||||
|
||||
void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
|
||||
|
||||
const StoragePtr & getStorage() const { return storage; }
|
||||
|
||||
private:
|
||||
static constexpr auto name = "ReadFromMemoryStorage";
|
||||
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
#include <Processors/Transforms/ColumnPermuteTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
void applyPermutation(std::vector<T> & data, const std::vector<size_t> & permutation)
|
||||
{
|
||||
std::vector<T> res;
|
||||
res.reserve(permutation.size());
|
||||
for (size_t i : permutation)
|
||||
res.push_back(data[i]);
|
||||
data = std::move(res);
|
||||
}
|
||||
|
||||
void permuteChunk(Chunk & chunk, const std::vector<size_t> & permutation)
|
||||
{
|
||||
size_t num_rows = chunk.getNumRows();
|
||||
auto columns = chunk.detachColumns();
|
||||
applyPermutation(columns, permutation);
|
||||
chunk.setColumns(std::move(columns), num_rows);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Block ColumnPermuteTransform::permute(const Block & block, const std::vector<size_t> & permutation)
|
||||
{
|
||||
auto columns = block.getColumnsWithTypeAndName();
|
||||
applyPermutation(columns, permutation);
|
||||
return Block(columns);
|
||||
}
|
||||
|
||||
ColumnPermuteTransform::ColumnPermuteTransform(const Block & header_, const std::vector<size_t> & permutation_)
|
||||
: ISimpleTransform(header_, permute(header_, permutation_), false)
|
||||
, permutation(permutation_)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void ColumnPermuteTransform::transform(Chunk & chunk)
|
||||
{
|
||||
permuteChunk(chunk, permutation);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <Interpreters/Set.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ColumnPermuteTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
ColumnPermuteTransform(const Block & header_, const std::vector<size_t> & permutation_);
|
||||
|
||||
String getName() const override { return "ColumnPermuteTransform"; }
|
||||
|
||||
void transform(Chunk & chunk) override;
|
||||
|
||||
static Block permute(const Block & block, const std::vector<size_t> & permutation);
|
||||
|
||||
private:
|
||||
Names column_names;
|
||||
std::vector<size_t> permutation;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
@ -7,18 +7,20 @@
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
|
||||
#include <Common/FieldVisitorScale.h>
|
||||
#include <Common/FieldVisitorSum.h>
|
||||
#include <Common/FieldVisitorToString.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
constexpr bool debug_logging_enabled = false;
|
||||
constexpr static bool debug_logging_enabled = false;
|
||||
|
||||
template <typename T>
|
||||
void logDebug(String key, const T & value, const char * separator = " : ")
|
||||
inline static void logDebug(const char * key, const T & value, const char * separator = " : ")
|
||||
{
|
||||
if constexpr (debug_logging_enabled)
|
||||
{
|
||||
@ -60,15 +62,74 @@ static FillColumnDescription::StepFunction getStepFunction(
|
||||
{
|
||||
#define DECLARE_CASE(NAME) \
|
||||
case IntervalKind::Kind::NAME: \
|
||||
return [step, scale, &date_lut](Field & field) { \
|
||||
return [step, scale, &date_lut](Field & field, Int32 jumps_count) { \
|
||||
field = Add##NAME##sImpl::execute(static_cast<T>(\
|
||||
field.safeGet<T>()), static_cast<Int32>(step), date_lut, utc_time_zone, scale); };
|
||||
field.safeGet<T>()), static_cast<Int32>(step) * jumps_count, date_lut, utc_time_zone, scale); };
|
||||
|
||||
FOR_EACH_INTERVAL_KIND(DECLARE_CASE)
|
||||
#undef DECLARE_CASE
|
||||
}
|
||||
}
|
||||
|
||||
static FillColumnDescription::StepFunction getStepFunction(const Field & step, const std::optional<IntervalKind> & step_kind, const DataTypePtr & type)
|
||||
{
|
||||
WhichDataType which(type);
|
||||
|
||||
if (step_kind)
|
||||
{
|
||||
if (which.isDate() || which.isDate32())
|
||||
{
|
||||
Int64 avg_seconds = step.safeGet<Int64>() * step_kind->toAvgSeconds();
|
||||
if (std::abs(avg_seconds) < 86400)
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"Value of step is to low ({} seconds). Must be >= 1 day", std::abs(avg_seconds));
|
||||
}
|
||||
|
||||
if (which.isDate())
|
||||
return getStepFunction<UInt16>(step_kind.value(), step.safeGet<Int64>(), DateLUT::instance());
|
||||
else if (which.isDate32())
|
||||
return getStepFunction<Int32>(step_kind.value(), step.safeGet<Int64>(), DateLUT::instance());
|
||||
else if (const auto * date_time = checkAndGetDataType<DataTypeDateTime>(type.get()))
|
||||
return getStepFunction<UInt32>(step_kind.value(), step.safeGet<Int64>(), date_time->getTimeZone());
|
||||
else if (const auto * date_time64 = checkAndGetDataType<DataTypeDateTime64>(type.get()))
|
||||
{
|
||||
const auto & step_dec = step.safeGet<const DecimalField<Decimal64> &>();
|
||||
Int64 converted_step = DecimalUtils::convertTo<Int64>(step_dec.getValue(), step_dec.getScale());
|
||||
static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
|
||||
|
||||
switch (step_kind.value()) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
#define DECLARE_CASE(NAME) \
|
||||
case IntervalKind::Kind::NAME: \
|
||||
return [converted_step, &time_zone = date_time64->getTimeZone()](Field & field, Int32 jumps_count) \
|
||||
{ \
|
||||
auto field_decimal = field.safeGet<DecimalField<DateTime64>>(); \
|
||||
auto res = Add##NAME##sImpl::execute(field_decimal.getValue(), converted_step * jumps_count, time_zone, utc_time_zone, field_decimal.getScale()); \
|
||||
field = DecimalField(res, field_decimal.getScale()); \
|
||||
}; \
|
||||
break;
|
||||
|
||||
FOR_EACH_INTERVAL_KIND(DECLARE_CASE)
|
||||
#undef DECLARE_CASE
|
||||
}
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName());
|
||||
}
|
||||
else
|
||||
{
|
||||
return [step](Field & field, Int32 jumps_count)
|
||||
{
|
||||
auto shifted_step = step;
|
||||
if (jumps_count != 1)
|
||||
applyVisitor(FieldVisitorScale(jumps_count), shifted_step);
|
||||
|
||||
applyVisitor(FieldVisitorSum(shifted_step), field);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & type)
|
||||
{
|
||||
auto max_type = Field::Types::Null;
|
||||
@ -125,7 +186,8 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr &
|
||||
|
||||
if (descr.fill_from.getType() > max_type
|
||||
|| descr.fill_to.getType() > max_type
|
||||
|| descr.fill_step.getType() > max_type)
|
||||
|| descr.fill_step.getType() > max_type
|
||||
|| descr.fill_staleness.getType() > max_type)
|
||||
return false;
|
||||
|
||||
if (!descr.fill_from.isNull())
|
||||
@ -134,56 +196,11 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr &
|
||||
descr.fill_to = convertFieldToTypeOrThrow(descr.fill_to, *to_type);
|
||||
if (!descr.fill_step.isNull())
|
||||
descr.fill_step = convertFieldToTypeOrThrow(descr.fill_step, *to_type);
|
||||
if (!descr.fill_staleness.isNull())
|
||||
descr.fill_staleness = convertFieldToTypeOrThrow(descr.fill_staleness, *to_type);
|
||||
|
||||
if (descr.step_kind)
|
||||
{
|
||||
if (which.isDate() || which.isDate32())
|
||||
{
|
||||
Int64 avg_seconds = descr.fill_step.safeGet<Int64>() * descr.step_kind->toAvgSeconds();
|
||||
if (std::abs(avg_seconds) < 86400)
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"Value of step is to low ({} seconds). Must be >= 1 day", std::abs(avg_seconds));
|
||||
}
|
||||
|
||||
if (which.isDate())
|
||||
descr.step_func = getStepFunction<UInt16>(*descr.step_kind, descr.fill_step.safeGet<Int64>(), DateLUT::instance());
|
||||
else if (which.isDate32())
|
||||
descr.step_func = getStepFunction<Int32>(*descr.step_kind, descr.fill_step.safeGet<Int64>(), DateLUT::instance());
|
||||
else if (const auto * date_time = checkAndGetDataType<DataTypeDateTime>(type.get()))
|
||||
descr.step_func = getStepFunction<UInt32>(*descr.step_kind, descr.fill_step.safeGet<Int64>(), date_time->getTimeZone());
|
||||
else if (const auto * date_time64 = checkAndGetDataType<DataTypeDateTime64>(type.get()))
|
||||
{
|
||||
const auto & step_dec = descr.fill_step.safeGet<const DecimalField<Decimal64> &>();
|
||||
Int64 step = DecimalUtils::convertTo<Int64>(step_dec.getValue(), step_dec.getScale());
|
||||
static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
|
||||
|
||||
switch (*descr.step_kind) // NOLINT(bugprone-switch-missing-default-case)
|
||||
{
|
||||
#define DECLARE_CASE(NAME) \
|
||||
case IntervalKind::Kind::NAME: \
|
||||
descr.step_func = [step, &time_zone = date_time64->getTimeZone()](Field & field) \
|
||||
{ \
|
||||
auto field_decimal = field.safeGet<DecimalField<DateTime64>>(); \
|
||||
auto res = Add##NAME##sImpl::execute(field_decimal.getValue(), step, time_zone, utc_time_zone, field_decimal.getScale()); \
|
||||
field = DecimalField(res, field_decimal.getScale()); \
|
||||
}; \
|
||||
break;
|
||||
|
||||
FOR_EACH_INTERVAL_KIND(DECLARE_CASE)
|
||||
#undef DECLARE_CASE
|
||||
}
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
|
||||
"STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName());
|
||||
}
|
||||
else
|
||||
{
|
||||
descr.step_func = [step = descr.fill_step](Field & field)
|
||||
{
|
||||
applyVisitor(FieldVisitorSum(step), field);
|
||||
};
|
||||
}
|
||||
descr.step_func = getStepFunction(descr.fill_step, descr.step_kind, type);
|
||||
descr.staleness_step_func = getStepFunction(descr.fill_staleness, descr.staleness_kind, type);
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -218,6 +235,7 @@ FillingTransform::FillingTransform(
|
||||
fill_column_positions.push_back(block_position);
|
||||
|
||||
auto & descr = filling_row.getFillDescription(i);
|
||||
running_with_staleness |= !descr.fill_staleness.isNull();
|
||||
|
||||
const Block & output_header = getOutputPort().getHeader();
|
||||
const DataTypePtr & type = removeNullable(output_header.getByPosition(block_position).type);
|
||||
@ -437,7 +455,7 @@ void FillingTransform::initColumns(
|
||||
non_const_columns.reserve(input_columns.size());
|
||||
|
||||
for (const auto & column : input_columns)
|
||||
non_const_columns.push_back(column->convertToFullColumnIfConst());
|
||||
non_const_columns.push_back(column->convertToFullColumnIfConst()->convertToFullColumnIfSparse());
|
||||
|
||||
for (const auto & column : non_const_columns)
|
||||
output_columns.push_back(column->cloneEmpty()->assumeMutable());
|
||||
@ -482,26 +500,26 @@ bool FillingTransform::generateSuffixIfNeeded(
|
||||
MutableColumnRawPtrs res_sort_prefix_columns,
|
||||
MutableColumnRawPtrs res_other_columns)
|
||||
{
|
||||
logDebug("generateSuffixIfNeeded() filling_row", filling_row);
|
||||
logDebug("generateSuffixIfNeeded() next_row", next_row);
|
||||
logDebug("generateSuffixIfNeeded filling_row", filling_row);
|
||||
logDebug("generateSuffixIfNeeded next_row", next_row);
|
||||
|
||||
/// Determines if we should insert filling row before start generating next rows
|
||||
bool should_insert_first = (next_row < filling_row && !filling_row_inserted) || next_row.isNull();
|
||||
bool should_insert_first = (next_row < filling_row && !filling_row_inserted) || (next_row.isNull() && !filling_row.isNull());
|
||||
logDebug("should_insert_first", should_insert_first);
|
||||
|
||||
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
|
||||
next_row[i] = filling_row.getFillDescription(i).fill_to;
|
||||
next_row[i] = Field{};
|
||||
|
||||
logDebug("generateSuffixIfNeeded() next_row updated", next_row);
|
||||
logDebug("generateSuffixIfNeeded next_row updated", next_row);
|
||||
|
||||
if (filling_row >= next_row)
|
||||
if (!filling_row.hasSomeConstraints() || !filling_row.isConstraintsSatisfied())
|
||||
{
|
||||
logDebug("generateSuffixIfNeeded()", "no need to generate suffix");
|
||||
logDebug("generateSuffixIfNeeded", "will not generate suffix");
|
||||
return false;
|
||||
}
|
||||
|
||||
Block interpolate_block;
|
||||
if (should_insert_first && filling_row < next_row)
|
||||
if (should_insert_first)
|
||||
{
|
||||
interpolate(result_columns, interpolate_block);
|
||||
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block);
|
||||
@ -516,9 +534,7 @@ bool FillingTransform::generateSuffixIfNeeded(
|
||||
bool filling_row_changed = false;
|
||||
while (true)
|
||||
{
|
||||
const auto [apply, changed] = filling_row.next(next_row);
|
||||
filling_row_changed = changed;
|
||||
if (!apply)
|
||||
if (!filling_row.next(next_row, filling_row_changed))
|
||||
break;
|
||||
|
||||
interpolate(result_columns, interpolate_block);
|
||||
@ -595,7 +611,7 @@ void FillingTransform::transformRange(
|
||||
|
||||
if (!fill_from.isNull() && !equals(current_value, fill_from))
|
||||
{
|
||||
filling_row.initFromDefaults(i);
|
||||
filling_row.initUsingFrom(i);
|
||||
filling_row_inserted = false;
|
||||
if (less(fill_from, current_value, filling_row.getDirection(i)))
|
||||
{
|
||||
@ -609,6 +625,9 @@ void FillingTransform::transformRange(
|
||||
}
|
||||
}
|
||||
|
||||
/// Init staleness first interval
|
||||
filling_row.updateConstraintsWithStalenessRow(input_fill_columns, range_begin);
|
||||
|
||||
for (size_t row_ind = range_begin; row_ind < range_end; ++row_ind)
|
||||
{
|
||||
logDebug("row", row_ind);
|
||||
@ -619,21 +638,14 @@ void FillingTransform::transformRange(
|
||||
logDebug("should_insert_first", should_insert_first);
|
||||
|
||||
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
|
||||
{
|
||||
const auto current_value = (*input_fill_columns[i])[row_ind];
|
||||
const auto & fill_to = filling_row.getFillDescription(i).fill_to;
|
||||
next_row[i] = (*input_fill_columns[i])[row_ind];
|
||||
|
||||
if (fill_to.isNull() || less(current_value, fill_to, filling_row.getDirection(i)))
|
||||
next_row[i] = current_value;
|
||||
else
|
||||
next_row[i] = fill_to;
|
||||
}
|
||||
logDebug("next_row updated", next_row);
|
||||
|
||||
/// The condition is true when filling row is initialized by value(s) in FILL FROM,
|
||||
/// and there are row(s) in current range with value(s) < then in the filling row.
|
||||
/// It can happen only once for a range.
|
||||
if (should_insert_first && filling_row < next_row)
|
||||
if (should_insert_first && filling_row < next_row && filling_row.isConstraintsSatisfied())
|
||||
{
|
||||
interpolate(result_columns, interpolate_block);
|
||||
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block);
|
||||
@ -643,15 +655,37 @@ void FillingTransform::transformRange(
|
||||
bool filling_row_changed = false;
|
||||
while (true)
|
||||
{
|
||||
const auto [apply, changed] = filling_row.next(next_row);
|
||||
filling_row_changed = changed;
|
||||
if (!apply)
|
||||
if (!filling_row.next(next_row, filling_row_changed))
|
||||
break;
|
||||
|
||||
interpolate(result_columns, interpolate_block);
|
||||
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block);
|
||||
copyRowFromColumns(res_sort_prefix_columns, input_sort_prefix_columns, row_ind);
|
||||
filling_row_changed = false;
|
||||
}
|
||||
|
||||
if (running_with_staleness)
|
||||
{
|
||||
/// Initialize staleness border for current row to generate it's prefix
|
||||
filling_row.updateConstraintsWithStalenessRow(input_fill_columns, row_ind);
|
||||
|
||||
while (filling_row.shift(next_row, filling_row_changed))
|
||||
{
|
||||
logDebug("filling_row after shift", filling_row);
|
||||
|
||||
do
|
||||
{
|
||||
logDebug("inserting prefix filling_row", filling_row);
|
||||
|
||||
interpolate(result_columns, interpolate_block);
|
||||
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, interpolate_block);
|
||||
copyRowFromColumns(res_sort_prefix_columns, input_sort_prefix_columns, row_ind);
|
||||
filling_row_changed = false;
|
||||
|
||||
} while (filling_row.next(next_row, filling_row_changed));
|
||||
}
|
||||
}
|
||||
|
||||
/// new valid filling row was generated but not inserted, will use it during suffix generation
|
||||
if (filling_row_changed)
|
||||
filling_row_inserted = false;
|
||||
@ -707,7 +741,7 @@ void FillingTransform::transform(Chunk & chunk)
|
||||
/// if no data was processed, then need to initialize filling_row
|
||||
if (last_row.empty())
|
||||
{
|
||||
filling_row.initFromDefaults();
|
||||
filling_row.initUsingFrom();
|
||||
filling_row_inserted = false;
|
||||
}
|
||||
|
||||
|
||||
@ -84,6 +84,7 @@ private:
|
||||
SortDescription sort_prefix;
|
||||
const InterpolateDescriptionPtr interpolate_description; /// Contains INTERPOLATE columns
|
||||
|
||||
bool running_with_staleness = false; /// True if STALENESS clause was used.
|
||||
FillingRow filling_row; /// Current row, which is used to fill gaps.
|
||||
FillingRow next_row; /// Row to which we need to generate filling rows.
|
||||
bool filling_row_inserted = false;
|
||||
|
||||
@ -19,7 +19,6 @@ Block JoiningTransform::transformHeader(Block header, const JoinPtr & join)
|
||||
join->initialize(header);
|
||||
ExtraBlockPtr tmp;
|
||||
join->joinBlock(header, tmp);
|
||||
materializeBlockInplace(header);
|
||||
LOG_TEST(getLogger("JoiningTransform"), "After join block: '{}'", header.dumpStructure());
|
||||
return header;
|
||||
}
|
||||
|
||||
@ -394,7 +394,7 @@ void FullMergeJoinCursor::setChunk(Chunk && chunk)
|
||||
convertToFullIfSparse(chunk);
|
||||
|
||||
current_chunk = std::move(chunk);
|
||||
cursor = SortCursorImpl(sample_block, current_chunk.getColumns(), desc);
|
||||
cursor = SortCursorImpl(sample_block, current_chunk.getColumns(), current_chunk.getNumRows(), desc);
|
||||
}
|
||||
|
||||
bool FullMergeJoinCursor::fullyCompleted() const
|
||||
|
||||
@ -42,7 +42,7 @@ MergeSorter::MergeSorter(const Block & header, Chunks chunks_, SortDescription &
|
||||
/// Convert to full column, because some cursors expect non-contant columns
|
||||
convertToFullIfConst(chunk);
|
||||
|
||||
cursors.emplace_back(header, chunk.getColumns(), description, chunk_index);
|
||||
cursors.emplace_back(header, chunk.getColumns(), chunk.getNumRows(), description, chunk_index);
|
||||
has_collation |= cursors.back().has_collation;
|
||||
|
||||
nonempty_chunks.emplace_back(std::move(chunk));
|
||||
|
||||
@ -1496,7 +1496,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
|
||||
if (command.to_remove == AlterCommand::RemoveProperty::CODEC && column_from_table.codec == nullptr)
|
||||
throw Exception(
|
||||
ErrorCodes::BAD_ARGUMENTS,
|
||||
"Column {} doesn't have TTL, cannot remove it",
|
||||
"Column {} doesn't have CODEC, cannot remove it",
|
||||
backQuote(column_name));
|
||||
if (command.to_remove == AlterCommand::RemoveProperty::COMMENT && column_from_table.comment.empty())
|
||||
throw Exception(
|
||||
|
||||
@ -151,6 +151,18 @@ KeyDescription KeyDescription::getSortingKeyFromAST(
|
||||
throw Exception(ErrorCodes::DATA_TYPE_CANNOT_BE_USED_IN_KEY,
|
||||
"Column {} with type {} is not allowed in key expression, it's not comparable",
|
||||
backQuote(result.sample_block.getByPosition(i).name), result.data_types.back()->getName());
|
||||
|
||||
auto check = [&](const IDataType & type)
|
||||
{
|
||||
if (isDynamic(type) || isVariant(type))
|
||||
throw Exception(
|
||||
ErrorCodes::DATA_TYPE_CANNOT_BE_USED_IN_KEY,
|
||||
"Column with type Variant/Dynamic is not allowed in key expression. Consider using a subcolumn with a specific data "
|
||||
"type instead (for example 'column.Int64' or 'json.some.path.:Int64' if its a JSON path subcolumn) or casting this column to a specific data type");
|
||||
};
|
||||
|
||||
check(*result.data_types.back());
|
||||
result.data_types.back()->forEachChild(check);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@ -48,6 +48,16 @@ namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric BackgroundMergesAndMutationsPoolTask;
|
||||
}
|
||||
namespace ProfileEvents
|
||||
{
|
||||
|
||||
extern const Event MergerMutatorsGetPartsForMergeElapsedMicroseconds;
|
||||
extern const Event MergerMutatorPrepareRangesForMergeElapsedMicroseconds;
|
||||
extern const Event MergerMutatorSelectPartsForMergeElapsedMicroseconds;
|
||||
extern const Event MergerMutatorRangesForMergeCount;
|
||||
extern const Event MergerMutatorPartsInRangesForMergeCount;
|
||||
extern const Event MergerMutatorSelectRangePartsCount;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -215,6 +225,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
|
||||
{
|
||||
PartitionIdsHint res;
|
||||
MergeTreeData::DataPartsVector data_parts = getDataPartsToSelectMergeFrom(txn);
|
||||
|
||||
if (data_parts.empty())
|
||||
return res;
|
||||
|
||||
@ -272,6 +283,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
|
||||
MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectMergeFrom(
|
||||
const MergeTreeTransactionPtr & txn, const PartitionIdsHint * partitions_hint) const
|
||||
{
|
||||
|
||||
Stopwatch get_data_parts_for_merge_timer;
|
||||
auto res = getDataPartsToSelectMergeFrom(txn);
|
||||
if (!partitions_hint)
|
||||
return res;
|
||||
@ -280,6 +293,8 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::getDataPartsToSelectM
|
||||
{
|
||||
return !partitions_hint->contains(part->info.partition_id);
|
||||
});
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorsGetPartsForMergeElapsedMicroseconds, get_data_parts_for_merge_timer.elapsedMicroseconds());
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -357,6 +372,7 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
PreformattedMessage & out_disable_reason) const
|
||||
{
|
||||
Stopwatch ranges_for_merge_timer;
|
||||
MergeSelectingInfo res;
|
||||
|
||||
res.current_time = std::time(nullptr);
|
||||
@ -457,6 +473,10 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
|
||||
prev_part = ∂
|
||||
}
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorPartsInRangesForMergeCount, res.parts_selected_precondition);
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorRangesForMergeCount, res.parts_ranges.size());
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorPrepareRangesForMergeElapsedMicroseconds, ranges_for_merge_timer.elapsedMicroseconds());
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -471,6 +491,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool dry_run)
|
||||
{
|
||||
Stopwatch select_parts_from_ranges_timer;
|
||||
const auto data_settings = data.getSettings();
|
||||
IMergeSelector::PartsRange parts_to_merge;
|
||||
|
||||
@ -570,7 +591,8 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
|
||||
|
||||
if (parts_to_merge.empty())
|
||||
{
|
||||
out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors)");
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds());
|
||||
out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors) in {}ms", select_parts_from_ranges_timer.elapsedMicroseconds() / 1000);
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
}
|
||||
@ -583,8 +605,11 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
|
||||
parts.push_back(part);
|
||||
}
|
||||
|
||||
LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name);
|
||||
LOG_DEBUG(log, "Selected {} parts from {} to {} in {}ms", parts.size(), parts.front()->name, parts.back()->name, select_parts_from_ranges_timer.elapsedMicroseconds() / 1000);
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorSelectRangePartsCount, parts.size());
|
||||
|
||||
future_part->assign(std::move(parts));
|
||||
ProfileEvents::increment(ProfileEvents::MergerMutatorSelectPartsForMergeElapsedMicroseconds, select_parts_from_ranges_timer.elapsedMicroseconds());
|
||||
return SelectPartsDecision::SELECTED;
|
||||
}
|
||||
|
||||
|
||||
@ -345,10 +345,11 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Index granularity is too big: more than {} rows per index granule.", std::numeric_limits<UInt32>::max());
|
||||
|
||||
if (index_sample_block.columns() > 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected block with single column");
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected that index is build over a single column");
|
||||
|
||||
const String & index_column_name = index_sample_block.getByPosition(0).name;
|
||||
const ColumnPtr & index_column = block.getByName(index_column_name).column;
|
||||
const auto & index_column_name = index_sample_block.getByPosition(0).name;
|
||||
|
||||
const auto & index_column = block.getByName(index_column_name).column;
|
||||
ColumnPtr column_cut = index_column->cut(*pos, rows_read);
|
||||
|
||||
const auto * column_array = typeid_cast<const ColumnArray *>(column_cut.get());
|
||||
@ -382,8 +383,7 @@ void MergeTreeIndexAggregatorVectorSimilarity::update(const Block & block, size_
|
||||
if (index->size() + rows > std::numeric_limits<UInt32>::max())
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Size of vector similarity index would exceed 4 billion entries");
|
||||
|
||||
DataTypePtr data_type = block.getDataTypes()[0];
|
||||
const auto * data_type_array = typeid_cast<const DataTypeArray *>(data_type.get());
|
||||
const auto * data_type_array = typeid_cast<const DataTypeArray *>(block.getByName(index_column_name).type.get());
|
||||
if (!data_type_array)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected data type Array(Float*)");
|
||||
const TypeIndex nested_type_index = data_type_array->getNestedType()->getTypeId();
|
||||
|
||||
@ -30,8 +30,8 @@ ColumnsDescription StorageSystemGrants::getColumnsDescription()
|
||||
{"column", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>()), "Name of a column to which access is granted."},
|
||||
{"is_partial_revoke", std::make_shared<DataTypeUInt8>(),
|
||||
"Logical value. It shows whether some privileges have been revoked. Possible values: "
|
||||
"0 — The row describes a partial revoke, "
|
||||
"1 — The row describes a grant."
|
||||
"0 — The row describes a grant, "
|
||||
"1 — The row describes a partial revoke."
|
||||
},
|
||||
{"grant_option", std::make_shared<DataTypeUInt8>(), "Permission is granted WITH GRANT OPTION."},
|
||||
};
|
||||
|
||||
@ -51,11 +51,11 @@ class CI:
|
||||
|
||||
TAG_CONFIGS = {
|
||||
Tags.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
|
||||
Tags.CI_SET_ARM: LabelConfig(
|
||||
Tags.CI_SET_AARCH64: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
BuildNames.PACKAGE_AARCH64,
|
||||
JobNames.INTEGRATION_TEST_ARM,
|
||||
JobNames.INTEGRATION_TEST_AARCH64,
|
||||
]
|
||||
),
|
||||
Tags.CI_SET_REQUIRED: LabelConfig(
|
||||
@ -95,16 +95,16 @@ class CI:
|
||||
static_binary_name="aarch64",
|
||||
additional_pkgs=True,
|
||||
),
|
||||
runner_type=Runners.BUILDER_ARM,
|
||||
runner_type=Runners.BUILDER_AARCH64,
|
||||
),
|
||||
BuildNames.PACKAGE_ARM_ASAN: CommonJobConfigs.BUILD.with_properties(
|
||||
BuildNames.PACKAGE_AARCH64_ASAN: CommonJobConfigs.BUILD.with_properties(
|
||||
build_config=BuildConfig(
|
||||
name=BuildNames.PACKAGE_ARM_ASAN,
|
||||
name=BuildNames.PACKAGE_AARCH64_ASAN,
|
||||
compiler="clang-18-aarch64",
|
||||
sanitizer="address",
|
||||
package_type="deb",
|
||||
),
|
||||
runner_type=Runners.BUILDER_ARM,
|
||||
runner_type=Runners.BUILDER_AARCH64,
|
||||
),
|
||||
BuildNames.PACKAGE_ASAN: CommonJobConfigs.BUILD.with_properties(
|
||||
build_config=BuildConfig(
|
||||
@ -276,16 +276,16 @@ class CI:
|
||||
JobNames.INSTALL_TEST_AMD: CommonJobConfigs.INSTALL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE]
|
||||
),
|
||||
JobNames.INSTALL_TEST_ARM: CommonJobConfigs.INSTALL_TEST.with_properties(
|
||||
JobNames.INSTALL_TEST_AARCH64: CommonJobConfigs.INSTALL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
),
|
||||
JobNames.STATEFUL_TEST_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN]
|
||||
),
|
||||
JobNames.STATEFUL_TEST_ARM_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ARM_ASAN],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
JobNames.STATEFUL_TEST_AARCH64_ASAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64_ASAN],
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.STATEFUL_TEST_TSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_TSAN]
|
||||
@ -307,7 +307,7 @@ class CI:
|
||||
),
|
||||
JobNames.STATEFUL_TEST_AARCH64: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.STATEFUL_TEST_PARALLEL_REPL_RELEASE: CommonJobConfigs.STATEFUL_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE]
|
||||
@ -335,10 +335,10 @@ class CI:
|
||||
JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
|
||||
),
|
||||
JobNames.STATELESS_TEST_ARM_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_ARM_ASAN],
|
||||
JobNames.STATELESS_TEST_AARCH64_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64_ASAN],
|
||||
num_batches=2,
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4
|
||||
@ -360,7 +360,7 @@ class CI:
|
||||
),
|
||||
JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=2
|
||||
@ -432,10 +432,10 @@ class CI:
|
||||
num_batches=6,
|
||||
timeout=9000, # the job timed out with default value (7200)
|
||||
),
|
||||
JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties(
|
||||
JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
num_batches=6,
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE],
|
||||
@ -453,10 +453,10 @@ class CI:
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE],
|
||||
required_on_release_branch=True,
|
||||
),
|
||||
JobNames.COMPATIBILITY_TEST_ARM: CommonJobConfigs.COMPATIBILITY_TEST.with_properties(
|
||||
JobNames.COMPATIBILITY_TEST_AARCH64: CommonJobConfigs.COMPATIBILITY_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
required_on_release_branch=True,
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
),
|
||||
JobNames.UNIT_TEST: CommonJobConfigs.UNIT_TEST.with_properties(
|
||||
required_builds=[BuildNames.BINARY_RELEASE],
|
||||
@ -499,22 +499,22 @@ class CI:
|
||||
required_builds=[BuildNames.BINARY_RELEASE],
|
||||
run_by_labels=[Labels.JEPSEN_TEST],
|
||||
run_command="jepsen_check.py keeper",
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
),
|
||||
JobNames.JEPSEN_SERVER: JobConfig(
|
||||
required_builds=[BuildNames.BINARY_RELEASE],
|
||||
run_by_labels=[Labels.JEPSEN_TEST],
|
||||
run_command="jepsen_check.py server",
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
),
|
||||
JobNames.PERFORMANCE_TEST_AMD64: CommonJobConfigs.PERF_TESTS.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
|
||||
),
|
||||
JobNames.PERFORMANCE_TEST_ARM64: CommonJobConfigs.PERF_TESTS.with_properties(
|
||||
JobNames.PERFORMANCE_TEST_AARCH64: CommonJobConfigs.PERF_TESTS.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
num_batches=4,
|
||||
run_by_labels=[Labels.PR_PERFORMANCE],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.SQLANCER: CommonJobConfigs.SQLLANCER_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE],
|
||||
@ -532,9 +532,9 @@ class CI:
|
||||
JobNames.CLICKBENCH_TEST: CommonJobConfigs.CLICKBENCH_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_RELEASE],
|
||||
),
|
||||
JobNames.CLICKBENCH_TEST_ARM: CommonJobConfigs.CLICKBENCH_TEST.with_properties(
|
||||
JobNames.CLICKBENCH_TEST_AARCH64: CommonJobConfigs.CLICKBENCH_TEST.with_properties(
|
||||
required_builds=[BuildNames.PACKAGE_AARCH64],
|
||||
runner_type=Runners.FUNC_TESTER_ARM,
|
||||
runner_type=Runners.FUNC_TESTER_AARCH64,
|
||||
),
|
||||
JobNames.LIBFUZZER_TEST: JobConfig(
|
||||
required_builds=[BuildNames.FUZZERS],
|
||||
@ -572,7 +572,7 @@ class CI:
|
||||
),
|
||||
JobNames.STYLE_CHECK: JobConfig(
|
||||
run_always=True,
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
),
|
||||
JobNames.BUGFIX_VALIDATE: JobConfig(
|
||||
run_by_labels=[Labels.PR_BUGFIX, Labels.PR_CRITICAL_BUGFIX],
|
||||
|
||||
@ -58,11 +58,11 @@ class Runners(metaclass=WithIter):
|
||||
"""
|
||||
|
||||
BUILDER = "builder"
|
||||
BUILDER_ARM = "builder-aarch64"
|
||||
BUILDER_AARCH64 = "builder-aarch64"
|
||||
STYLE_CHECKER = "style-checker"
|
||||
STYLE_CHECKER_ARM = "style-checker-aarch64"
|
||||
STYLE_CHECKER_AARCH64 = "style-checker-aarch64"
|
||||
FUNC_TESTER = "func-tester"
|
||||
FUNC_TESTER_ARM = "func-tester-aarch64"
|
||||
FUNC_TESTER_AARCH64 = "func-tester-aarch64"
|
||||
FUZZER_UNIT_TESTER = "fuzzer-unit-tester"
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ class Tags(metaclass=WithIter):
|
||||
# to upload all binaries from build jobs
|
||||
UPLOAD_ALL_ARTIFACTS = "upload_all"
|
||||
CI_SET_SYNC = "ci_set_sync"
|
||||
CI_SET_ARM = "ci_set_arm"
|
||||
CI_SET_AARCH64 = "ci_set_aarch64"
|
||||
CI_SET_REQUIRED = "ci_set_required"
|
||||
CI_SET_BUILDS = "ci_set_builds"
|
||||
|
||||
@ -106,7 +106,7 @@ class BuildNames(metaclass=WithIter):
|
||||
PACKAGE_MSAN = "package_msan"
|
||||
PACKAGE_DEBUG = "package_debug"
|
||||
PACKAGE_AARCH64 = "package_aarch64"
|
||||
PACKAGE_ARM_ASAN = "package_aarch64_asan"
|
||||
PACKAGE_AARCH64_ASAN = "package_aarch64_asan"
|
||||
PACKAGE_RELEASE_COVERAGE = "package_release_coverage"
|
||||
BINARY_RELEASE = "binary_release"
|
||||
BINARY_TIDY = "binary_tidy"
|
||||
@ -134,14 +134,14 @@ class JobNames(metaclass=WithIter):
|
||||
DOCKER_SERVER = "Docker server image"
|
||||
DOCKER_KEEPER = "Docker keeper image"
|
||||
INSTALL_TEST_AMD = "Install packages (release)"
|
||||
INSTALL_TEST_ARM = "Install packages (aarch64)"
|
||||
INSTALL_TEST_AARCH64 = "Install packages (aarch64)"
|
||||
|
||||
STATELESS_TEST_DEBUG = "Stateless tests (debug)"
|
||||
STATELESS_TEST_RELEASE = "Stateless tests (release)"
|
||||
STATELESS_TEST_RELEASE_COVERAGE = "Stateless tests (coverage)"
|
||||
STATELESS_TEST_AARCH64 = "Stateless tests (aarch64)"
|
||||
STATELESS_TEST_ASAN = "Stateless tests (asan)"
|
||||
STATELESS_TEST_ARM_ASAN = "Stateless tests (aarch64, asan)"
|
||||
STATELESS_TEST_AARCH64_ASAN = "Stateless tests (aarch64, asan)"
|
||||
STATELESS_TEST_TSAN = "Stateless tests (tsan)"
|
||||
STATELESS_TEST_MSAN = "Stateless tests (msan)"
|
||||
STATELESS_TEST_UBSAN = "Stateless tests (ubsan)"
|
||||
@ -158,7 +158,7 @@ class JobNames(metaclass=WithIter):
|
||||
STATEFUL_TEST_RELEASE_COVERAGE = "Stateful tests (coverage)"
|
||||
STATEFUL_TEST_AARCH64 = "Stateful tests (aarch64)"
|
||||
STATEFUL_TEST_ASAN = "Stateful tests (asan)"
|
||||
STATEFUL_TEST_ARM_ASAN = "Stateful tests (aarch64, asan)"
|
||||
STATEFUL_TEST_AARCH64_ASAN = "Stateful tests (aarch64, asan)"
|
||||
STATEFUL_TEST_TSAN = "Stateful tests (tsan)"
|
||||
STATEFUL_TEST_MSAN = "Stateful tests (msan)"
|
||||
STATEFUL_TEST_UBSAN = "Stateful tests (ubsan)"
|
||||
@ -181,7 +181,7 @@ class JobNames(metaclass=WithIter):
|
||||
INTEGRATION_TEST_ASAN = "Integration tests (asan)"
|
||||
INTEGRATION_TEST_ASAN_OLD_ANALYZER = "Integration tests (asan, old analyzer)"
|
||||
INTEGRATION_TEST_TSAN = "Integration tests (tsan)"
|
||||
INTEGRATION_TEST_ARM = "Integration tests (aarch64)"
|
||||
INTEGRATION_TEST_AARCH64 = "Integration tests (aarch64)"
|
||||
INTEGRATION_TEST_FLAKY = "Integration tests flaky check (asan)"
|
||||
|
||||
UPGRADE_TEST_DEBUG = "Upgrade check (debug)"
|
||||
@ -205,7 +205,7 @@ class JobNames(metaclass=WithIter):
|
||||
JEPSEN_SERVER = "ClickHouse Server Jepsen"
|
||||
|
||||
PERFORMANCE_TEST_AMD64 = "Performance Comparison (release)"
|
||||
PERFORMANCE_TEST_ARM64 = "Performance Comparison (aarch64)"
|
||||
PERFORMANCE_TEST_AARCH64 = "Performance Comparison (aarch64)"
|
||||
|
||||
# SQL_LOGIC_TEST = "Sqllogic test (release)"
|
||||
|
||||
@ -214,10 +214,10 @@ class JobNames(metaclass=WithIter):
|
||||
SQLTEST = "SQLTest"
|
||||
|
||||
COMPATIBILITY_TEST = "Compatibility check (release)"
|
||||
COMPATIBILITY_TEST_ARM = "Compatibility check (aarch64)"
|
||||
COMPATIBILITY_TEST_AARCH64 = "Compatibility check (aarch64)"
|
||||
|
||||
CLICKBENCH_TEST = "ClickBench (release)"
|
||||
CLICKBENCH_TEST_ARM = "ClickBench (aarch64)"
|
||||
CLICKBENCH_TEST_AARCH64 = "ClickBench (aarch64)"
|
||||
|
||||
LIBFUZZER_TEST = "libFuzzer tests"
|
||||
|
||||
@ -387,7 +387,7 @@ class CommonJobConfigs:
|
||||
"./tests/ci/upload_result_helper.py",
|
||||
],
|
||||
),
|
||||
runner_type=Runners.STYLE_CHECKER_ARM,
|
||||
runner_type=Runners.STYLE_CHECKER_AARCH64,
|
||||
disable_await=True,
|
||||
)
|
||||
COMPATIBILITY_TEST = JobConfig(
|
||||
@ -634,8 +634,8 @@ REQUIRED_CHECKS = [
|
||||
JobNames.STATEFUL_TEST_RELEASE,
|
||||
JobNames.STATELESS_TEST_RELEASE,
|
||||
JobNames.STATELESS_TEST_ASAN,
|
||||
JobNames.STATELESS_TEST_ARM_ASAN,
|
||||
JobNames.STATEFUL_TEST_ARM_ASAN,
|
||||
JobNames.STATELESS_TEST_AARCH64_ASAN,
|
||||
JobNames.STATEFUL_TEST_AARCH64_ASAN,
|
||||
JobNames.STATELESS_TEST_FLAKY_ASAN,
|
||||
JobNames.STATEFUL_TEST_ASAN,
|
||||
JobNames.STYLE_CHECK,
|
||||
|
||||
@ -131,7 +131,7 @@ def main():
|
||||
check_name = args.check_name or os.getenv("CHECK_NAME")
|
||||
assert check_name
|
||||
check_glibc = True
|
||||
# currently hardcoded to x86, don't enable for ARM
|
||||
# currently hardcoded to x86, don't enable for AARCH64
|
||||
check_distributions = (
|
||||
"aarch64" not in check_name.lower() and "arm64" not in check_name.lower()
|
||||
)
|
||||
|
||||
@ -36,11 +36,12 @@ class TestCIConfig(unittest.TestCase):
|
||||
elif "binary_" in job.lower() or "package_" in job.lower():
|
||||
if job.lower() in (
|
||||
CI.BuildNames.PACKAGE_AARCH64,
|
||||
CI.BuildNames.PACKAGE_ARM_ASAN,
|
||||
CI.BuildNames.PACKAGE_AARCH64_ASAN,
|
||||
):
|
||||
self.assertTrue(
|
||||
CI.JOB_CONFIGS[job].runner_type in (CI.Runners.BUILDER_ARM,),
|
||||
f"Job [{job}] must have [{CI.Runners.BUILDER_ARM}] runner",
|
||||
CI.JOB_CONFIGS[job].runner_type
|
||||
in (CI.Runners.BUILDER_AARCH64,),
|
||||
f"Job [{job}] must have [{CI.Runners.BUILDER_AARCH64}] runner",
|
||||
)
|
||||
else:
|
||||
self.assertTrue(
|
||||
@ -96,7 +97,7 @@ class TestCIConfig(unittest.TestCase):
|
||||
else:
|
||||
self.assertTrue(CI.JOB_CONFIGS[job].build_config is None)
|
||||
if "asan" in job and "aarch" in job:
|
||||
expected_builds = [CI.BuildNames.PACKAGE_ARM_ASAN]
|
||||
expected_builds = [CI.BuildNames.PACKAGE_AARCH64_ASAN]
|
||||
elif "asan" in job:
|
||||
expected_builds = [CI.BuildNames.PACKAGE_ASAN]
|
||||
elif "msan" in job:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user