thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into keeper-bench-2.0

Browse Source
This commit is contained in:
Antonio Andelic 2023-04-07 07:35:17 +00:00
commit 6e517c9251
80 changed files with 1569 additions and 711 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -1,4 +1,4 @@
[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/clickhouse-presentations/raw/master/images/logo-400x240.png)](https://clickhouse.com)
[<img alt="ClickHouse — open source distributed column-oriented DBMS" width="400px" src="https://clickhouse.com/images/ch_gh_logo_rounded.png" />](https://clickhouse.com?utm_source=github)
ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.

19
contrib/krb5-cmake/CMakeLists.txt
View File

@ -15,10 +15,6 @@ if(NOT AWK_PROGRAM)
message(FATAL_ERROR "You need the awk program to build ClickHouse with krb5 enabled.")
endif()
if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
add_compile_definitions(USE_BORINGSSL=1)
endif ()
set(KRB5_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/krb5/src")
set(KRB5_ET_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}/include_private")
@ -162,6 +158,11 @@ set(ALL_SRCS
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/kdf.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/cmac.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/des/des_keys.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/des/f_parity.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/enc_provider/rc4.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/hash_provider/hash_md4.c"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/md4/md4.c"
"${KRB5_SOURCE_DIR}/lib/crypto/krb/prng.c"
"${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_dk_cmac.c"
# "${KRB5_SOURCE_DIR}/lib/crypto/krb/crc32.c"
@ -226,7 +227,6 @@ set(ALL_SRCS
# "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des.c"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/rc4.c"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des3.c"
#"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/camellia.c"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/cmac.c"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/sha256.c"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/hmac.c"
@ -474,6 +474,14 @@ set(ALL_SRCS
"${KRB5_SOURCE_DIR}/lib/krb5/krb5_libinit.c"
)
if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
add_compile_definitions(USE_BORINGSSL=1)
else()
set(ALL_SRCS ${ALL_SRCS}
"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/camellia.c"
)
endif()
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/compile_et"
COMMAND /bin/sh
@ -673,6 +681,7 @@ target_include_directories(_krb5 PRIVATE
"${KRB5_SOURCE_DIR}/lib/gssapi/krb5"
"${KRB5_SOURCE_DIR}/lib/gssapi/spnego"
"${KRB5_SOURCE_DIR}/util/et"
"${KRB5_SOURCE_DIR}/lib/crypto/builtin/md4"
"${KRB5_SOURCE_DIR}/lib/crypto/openssl"
"${KRB5_SOURCE_DIR}/lib/crypto/krb"
"${KRB5_SOURCE_DIR}/util/profile"

137
docs/en/interfaces/formats.md
View File

@ -1936,30 +1936,31 @@ Setting `format_avro_schema_registry_url` needs to be configured in `users.xml`
The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries.
| Parquet data type (`INSERT`) | ClickHouse data type | Parquet data type (`SELECT`) |
|----------------------------------------------------|-----------------------------------------------------------------|------------------------------|
| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` |
| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` |
| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` |
| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` |
| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` |
| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` |
| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` |
| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` |
| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` |
| `FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT` |
| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `DOUBLE` |
| `DATE` | [Date32](/docs/en/sql-reference/data-types/date.md) | `DATE` |
| `TIME (ms)` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` |
| `TIMESTAMP`, `TIME (us, ns)` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `TIMESTAMP` |
| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` |
| `STRING`, `BINARY`, `FIXED_LENGTH_BYTE_ARRAY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_LENGTH_BYTE_ARRAY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| Parquet data type (`INSERT`) | ClickHouse data type | Parquet data type (`SELECT`) |
|-----------------------------------------------|------------------------------------------------------------------------------------------------------------|-------------------------------|
| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` |
| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` |
| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md)/[Enum8](/docs/en/sql-reference/data-types/enum.md) | `INT8` |
| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` |
| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md)/[Enum16](/docs/en/sql-reference/data-types/enum.md) | `INT16` |
| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` |
| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` |
| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` |
| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` |
| `FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT` |
| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `DOUBLE` |
| `DATE` | [Date32](/docs/en/sql-reference/data-types/date.md) | `DATE` |
| `TIME (ms)` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` |
| `TIMESTAMP`, `TIME (us, ns)` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `TIMESTAMP` |
| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` |
| `STRING`, `BINARY`, `FIXED_LENGTH_BYTE_ARRAY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_LENGTH_BYTE_ARRAY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@ -2005,31 +2006,32 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t
The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries.
| Arrow data type (`INSERT`) | ClickHouse data type | Arrow data type (`SELECT`) |
|-----------------------------------------|-----------------------------------------------------------------|----------------------------|
| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` |
| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` |
| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `INT8` |
| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` |
| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `INT16` |
| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` |
| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` |
| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` |
| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` |
| `FLOAT`, `HALF_FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT32` |
| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `FLOAT64` |
| `DATE32` | [Date32](/docs/en/sql-reference/data-types/date32.md) | `UINT16` |
| `DATE64` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` |
| `TIMESTAMP`, `TIME32`, `TIME64` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `UINT32` |
| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` |
| `STRING`, `BINARY`, `FIXED_SIZE_BINARY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_SIZE_BINARY` |
| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` |
| `DECIMAL256` | [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL256` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_SIZE_BINARY` |
| Arrow data type (`INSERT`) | ClickHouse data type | Arrow data type (`SELECT`) |
|-----------------------------------------|------------------------------------------------------------------------------------------------------------|----------------------------|
| `BOOL` | [Bool](/docs/en/sql-reference/data-types/boolean.md) | `BOOL` |
| `UINT8`, `BOOL` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `UINT8` |
| `INT8` | [Int8](/docs/en/sql-reference/data-types/int-uint.md)/[Enum8](/docs/en/sql-reference/data-types/enum.md) | `INT8` |
| `UINT16` | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) | `UINT16` |
| `INT16` | [Int16](/docs/en/sql-reference/data-types/int-uint.md)/[Enum16](/docs/en/sql-reference/data-types/enum.md) | `INT16` |
| `UINT32` | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `UINT32` |
| `INT32` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `INT32` |
| `UINT64` | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `UINT64` |
| `INT64` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `INT64` |
| `FLOAT`, `HALF_FLOAT` | [Float32](/docs/en/sql-reference/data-types/float.md) | `FLOAT32` |
| `DOUBLE` | [Float64](/docs/en/sql-reference/data-types/float.md) | `FLOAT64` |
| `DATE32` | [Date32](/docs/en/sql-reference/data-types/date32.md) | `UINT16` |
| `DATE64` | [DateTime](/docs/en/sql-reference/data-types/datetime.md) | `UINT32` |
| `TIMESTAMP`, `TIME32`, `TIME64` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `UINT32` |
| `STRING`, `BINARY` | [String](/docs/en/sql-reference/data-types/string.md) | `BINARY` |
| `STRING`, `BINARY`, `FIXED_SIZE_BINARY` | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) | `FIXED_SIZE_BINARY` |
| `DECIMAL` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL` |
| `DECIMAL256` | [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `DECIMAL256` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_SIZE_BINARY` |
| `FIXED_SIZE_BINARY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_SIZE_BINARY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@ -2078,23 +2080,26 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam
The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries.
| ORC data type (`INSERT`) | ClickHouse data type | ORC data type (`SELECT`) |
|---------------------------------------|---------------------------------------------------------------|--------------------------|
| `Boolean` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `Boolean` |
| `Tinyint` | [Int8](/docs/en/sql-reference/data-types/int-uint.md) | `Tinyint` |
| `Smallint` | [Int16](/docs/en/sql-reference/data-types/int-uint.md) | `Smallint` |
| `Int` | [Int32](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
| `Bigint` | [Int64](/docs/en/sql-reference/data-types/int-uint.md) | `Bigint` |
| `Float` | [Float32](/docs/en/sql-reference/data-types/float.md) | `Float` |
| `Double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `Double` |
| `Decimal` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `Decimal` |
| `Date` | [Date32](/docs/en/sql-reference/data-types/date32.md) | `Date` |
| `Timestamp` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `Timestamp` |
| `String`, `Char`, `Varchar`, `Binary` | [String](/docs/en/sql-reference/data-types/string.md) | `Binary` |
| `List` | [Array](/docs/en/sql-reference/data-types/array.md) | `List` |
| `Struct` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `Struct` |
| `Map` | [Map](/docs/en/sql-reference/data-types/map.md) | `Map` |
| `-` | [IPv4](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
| ORC data type (`INSERT`) | ClickHouse data type | ORC data type (`SELECT`) |
|---------------------------------------|-------------------------------------------------------------------------------------------------------------------|--------------------------|
| `Boolean` | [UInt8](/docs/en/sql-reference/data-types/int-uint.md) | `Boolean` |
| `Tinyint` | [Int8/UInt8](/docs/en/sql-reference/data-types/int-uint.md)/[Enum8](/docs/en/sql-reference/data-types/enum.md) | `Tinyint` |
| `Smallint` | [Int16/UInt16](/docs/en/sql-reference/data-types/int-uint.md)/[Enum16](/docs/en/sql-reference/data-types/enum.md) | `Smallint` |
| `Int` | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
| `Bigint` | [Int64/UInt32](/docs/en/sql-reference/data-types/int-uint.md) | `Bigint` |
| `Float` | [Float32](/docs/en/sql-reference/data-types/float.md) | `Float` |
| `Double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `Double` |
| `Decimal` | [Decimal](/docs/en/sql-reference/data-types/decimal.md) | `Decimal` |
| `Date` | [Date32](/docs/en/sql-reference/data-types/date32.md) | `Date` |
| `Timestamp` | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) | `Timestamp` |
| `String`, `Char`, `Varchar`, `Binary` | [String](/docs/en/sql-reference/data-types/string.md) | `Binary` |
| `List` | [Array](/docs/en/sql-reference/data-types/array.md) | `List` |
| `Struct` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `Struct` |
| `Map` | [Map](/docs/en/sql-reference/data-types/map.md) | `Map` |
| `Int` | [IPv4](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
| `Binary` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `Binary` |
| `Binary` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `Binary` |
| `Binary` | [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `Binary` |
Other types are not supported.

8
docs/en/operations/utilities/clickhouse-local.md
View File

@ -6,7 +6,13 @@ sidebar_label: clickhouse-local
# clickhouse-local
The `clickhouse-local` program enables you to perform fast processing on local files, without having to deploy and configure the ClickHouse server. It accepts data that represent tables and queries them using [ClickHouse SQL dialect](../../sql-reference/index.md). `clickhouse-local` uses the same core as ClickHouse server, so it supports most of the features and the same set of formats and table engines.
## When to use clickhouse-local vs. ClickHouse
`clickhouse-local` is an easy-to-use version of ClickHouse that is ideal for developers who need to perform fast processing on local and remote files using SQL without having to install a full database server. With `clickhouse-local`, developers can use SQL commands (using the [ClickHouse SQL dialect](../../sql-reference/index.md)) directly from the command line, providing a simple and efficient way to access ClickHouse features without the need for a full ClickHouse installation. One of the main benefits of `clickhouse-local` is that it is already included when installing [clickhouse-client](https://clickhouse.com/docs/en/integrations/sql-clients/clickhouse-client-local). This means that developers can get started with `clickhouse-local` quickly, without the need for a complex installation process.
While `clickhouse-local` is a great tool for development and testing purposes, and for processing files, it is not suitable for serving end users or applications. In these scenarios, it is recommended to use the open-source [ClickHouse](https://clickhouse.com/docs/en/install). ClickHouse is a powerful OLAP database that is designed to handle large-scale analytical workloads. It provides fast and efficient processing of complex queries on large datasets, making it ideal for use in production environments where high-performance is critical. Additionally, ClickHouse offers a wide range of features such as replication, sharding, and high availability, which are essential for scaling up to handle large datasets and serving applications. If you need to handle larger datasets or serve end users or applications, we recommend using open-source ClickHouse instead of `clickhouse-local`.
Please read the docs below that show example use cases for `clickhouse-local`, such as [querying local CSVs](#query-data-in-a-csv-file-using-sql) or [reading a parquet file in S3](#query-data-in-a-parquet-file-in-aws-s3).
## Download clickhouse-local

4
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -1276,7 +1276,7 @@ Using replacement fields, you can define a pattern for the resulting string. “
| %k | hour in 24h format (00-23) | 22 |
| %l | hour in 12h format (01-12) | 09 |
| %m | month as an integer number (01-12) | 01 |
| %M | full month name (January-December) | January |
| %M | full month name (January-December), see (*) below | January |
| %n | new-line character (‘’) | |
| %p | AM or PM designation | PM |
| %Q | Quarter (1-4) | 1 |
@ -1295,6 +1295,8 @@ Using replacement fields, you can define a pattern for the resulting string. “
| %z | Time offset from UTC as +HHMM or -HHMM | -0500 |
| %% | a % sign | % |
(*) In ClickHouse versions earlier than v23.4, `%M` prints the minute (00-59) instead of the full month name (January-December). The previous behavior can be restored using setting `formatdatetime_parsedatetime_m_is_month_name = 0`.
**Example**
Query:

4
docs/en/sql-reference/statements/drop.md
View File

@ -22,6 +22,10 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC]
Deletes the table.
:::tip
Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
:::
Syntax:
``` sql

99
docs/en/sql-reference/statements/undrop.md Normal file
View File

@ -0,0 +1,99 @@
---
slug: /en/sql-reference/statements/undrop
sidebar_label: UNDROP
---
# UNDROP TABLE
Cancels the dropping of the table.
Beginning with ClickHouse version 23.3 it is possible to UNDROP a table in an Atomic database
within `database_atomic_delay_before_drop_table_sec` (8 minutes by default) of issuing the DROP TABLE statement. Dropped tables are listed in
a system table called `system.dropped_tables`.
If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view.
:::note
UNDROP TABLE is experimental. To use it add this setting:
```sql
set allow_experimental_undrop_table_query = 1;
```
:::
:::tip
Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md)
:::
Syntax:
``` sql
UNDROP TABLE [db.]name [UUID '<uuid>'] [ON CLUSTER cluster]
```
**Example**
``` sql
set allow_experimental_undrop_table_query = 1;
```
```sql
CREATE TABLE undropMe
(
`id` UInt8
)
ENGINE = MergeTree
ORDER BY id
```
```sql
DROP TABLE undropMe
```
```sql
SELECT *
FROM system.dropped_tables
FORMAT Vertical
```
```response
Row 1:
──────
index: 0
database: default
table: undropMe
uuid: aa696a1a-1d70-4e60-a841-4c80827706cc
engine: MergeTree
metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql
table_dropped_time: 2023-04-05 14:12:12
1 row in set. Elapsed: 0.001 sec.
```
```sql
UNDROP TABLE undropMe
```
```response
Ok.
```
```sql
SELECT *
FROM system.dropped_tables
FORMAT Vertical
```
```response
Ok.
0 rows in set. Elapsed: 0.001 sec.
```
```sql
DESCRIBE TABLE undropMe
FORMAT Vertical
```
```response
Row 1:
──────
name: id
type: UInt8
default_type:
default_expression:
comment:
codec_expression:
ttl_expression:
```

141
src/Analyzer/Passes/QueryAnalysisPass.cpp
View File

@ -32,6 +32,7 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Formats/FormatFactory.h>
#include <Databases/IDatabase.h>
@ -75,6 +76,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/QueryTreeBuilder.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/Identifier.h>
namespace ProfileEvents
{
@ -112,6 +114,8 @@ namespace ErrorCodes
extern const int ALIAS_REQUIRED;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int UNKNOWN_TABLE;
extern const int ILLEGAL_COLUMN;
extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH;
}
/** Query analyzer implementation overview. Please check documentation in QueryAnalysisPass.h before.
@ -6079,6 +6083,18 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table
scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data));
}
bool findIdentifier(const FunctionNode & function)
{
for (const auto & argument : function.getArguments())
{
if (argument->as<IdentifierNode>())
return true;
if (const auto * f = argument->as<FunctionNode>(); f && findIdentifier(*f))
return true;
}
return false;
}
/// Resolve table function node in scope
void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
IdentifierResolveScope & scope,
@ -6090,12 +6106,11 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
if (!nested_table_function)
expressions_visitor.visit(table_function_node_typed.getArgumentsNode());
const auto & table_function_factory = TableFunctionFactory::instance();
const auto & table_function_name = table_function_node_typed.getTableFunctionName();
auto & scope_context = scope.context;
TableFunctionPtr table_function_ptr = table_function_factory.tryGet(table_function_name, scope_context);
TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().tryGet(table_function_name, scope_context);
if (!table_function_ptr)
{
auto hints = TableFunctionFactory::instance().getHints(table_function_name);
@ -6110,17 +6125,131 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
table_function_name);
}
uint64_t use_structure_from_insertion_table_in_table_functions = scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions;
if (!nested_table_function &&
scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions &&
use_structure_from_insertion_table_in_table_functions &&
scope_context->hasInsertionTable() &&
table_function_ptr->needStructureHint())
{
const auto & insertion_table = scope_context->getInsertionTable();
if (!insertion_table.empty())
{
auto insertion_table_storage = DatabaseCatalog::instance().getTable(insertion_table, scope_context);
const auto & structure_hint = insertion_table_storage->getInMemoryMetadataPtr()->columns;
table_function_ptr->setStructureHint(structure_hint);
const auto & insert_structure = DatabaseCatalog::instance().getTable(insertion_table, scope_context)->getInMemoryMetadataPtr()->getColumns();
DB::ColumnsDescription structure_hint;
bool use_columns_from_insert_query = true;
/// Insert table matches columns against SELECT expression by position, so we want to map
/// insert table columns to table function columns through names from SELECT expression.
auto insert_column = insert_structure.begin();
auto insert_structure_end = insert_structure.end(); /// end iterator of the range covered by possible asterisk
auto virtual_column_names = table_function_ptr->getVirtualsToCheckBeforeUsingStructureHint();
bool asterisk = false;
const auto & expression_list = scope.scope_node->as<QueryNode &>().getProjection();
auto expression = expression_list.begin();
/// We want to go through SELECT expression list and correspond each expression to column in insert table
/// which type will be used as a hint for the file structure inference.
for (; expression != expression_list.end() && insert_column != insert_structure_end; ++expression)
{
if (auto * identifier_node = (*expression)->as<IdentifierNode>())
{
if (!virtual_column_names.contains(identifier_node->getIdentifier().getFullName()))
{
if (asterisk)
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Asterisk cannot be mixed with column list in INSERT SELECT query.");
use_columns_from_insert_query = false;
break;
}
structure_hint.add({ identifier_node->getIdentifier().getFullName(), insert_column->type });
}
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
++insert_column;
}
else if (auto * matcher_node = (*expression)->as<MatcherNode>(); matcher_node && matcher_node->getMatcherType() == MatcherNodeType::ASTERISK)
{
if (asterisk)
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Only one asterisk can be used in INSERT SELECT query.");
use_columns_from_insert_query = false;
break;
}
if (!structure_hint.empty())
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Asterisk cannot be mixed with column list in INSERT SELECT query.");
use_columns_from_insert_query = false;
break;
}
asterisk = true;
}
else if (auto * function = (*expression)->as<FunctionNode>())
{
if (use_structure_from_insertion_table_in_table_functions == 2 && findIdentifier(*function))
{
use_columns_from_insert_query = false;
break;
}
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
++insert_column;
}
else
{
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
++insert_column;
}
}
if (use_structure_from_insertion_table_in_table_functions == 2 && !asterisk)
{
/// For input function we should check if input format supports reading subset of columns.
if (table_function_ptr->getName() == "input")
use_columns_from_insert_query = FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(scope.context->getInsertFormat());
else
use_columns_from_insert_query = table_function_ptr->supportsReadingSubsetOfColumns();
}
if (use_columns_from_insert_query)
{
if (expression == expression_list.end())
{
/// Append tail of insert structure to the hint
if (asterisk)
{
for (; insert_column != insert_structure_end; ++insert_column)
structure_hint.add({ insert_column->name, insert_column->type });
}
if (!structure_hint.empty())
table_function_ptr->setStructureHint(structure_hint);
} else if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns in insert table less than required by SELECT expression.");
}
}
}

41
src/Common/MemoryTracker.cpp
View File

@ -82,26 +82,6 @@ inline std::string_view toDescription(OvercommitResult result)
}
}
inline void debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused]])
{
/// Big allocations through allocNoThrow (without checking memory limits) may easily lead to OOM (and it's hard to debug).
/// Let's find them.
#ifdef ABORT_ON_LOGICAL_ERROR
if (size < 0)
return;
constexpr Int64 threshold = 16 * 1024 * 1024; /// The choice is arbitrary (maybe we should decrease it)
if (size < threshold)
return;
MemoryTrackerBlockerInThread blocker;
LOG_TEST(&Poco::Logger::get("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, "
"it may lead to OOM. Stack trace: {}", size, StackTrace().toString());
#else
return; /// Avoid trash logging in release builds
#endif
}
}
namespace ProfileEvents
@ -138,7 +118,6 @@ MemoryTracker::~MemoryTracker()
}
}
void MemoryTracker::logPeakMemoryUsage()
{
log_peak_memory_usage_in_destructor = false;
@ -176,6 +155,26 @@ void MemoryTracker::injectFault() const
description ? description : "");
}
void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused]])
{
/// Big allocations through allocNoThrow (without checking memory limits) may easily lead to OOM (and it's hard to debug).
/// Let's find them.
#ifdef ABORT_ON_LOGICAL_ERROR
if (size < 0)
return;
constexpr Int64 threshold = 16 * 1024 * 1024; /// The choice is arbitrary (maybe we should decrease it)
if (size < threshold)
return;
MemoryTrackerBlockerInThread blocker(VariableContext::Global);
LOG_TEST(&Poco::Logger::get("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, "
"it may lead to OOM. Stack trace: {}", size, StackTrace().toString());
#else
return; /// Avoid trash logging in release builds
#endif
}
void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker)
{
if (size < 0)

2
src/Common/MemoryTracker.h
View File

@ -215,6 +215,8 @@ public:
/// Prints info about peak memory consumption into log.
void logPeakMemoryUsage();
void debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused]]);
};
extern MemoryTracker total_memory_tracker;

10
src/Common/ThreadStatus.h
View File

@ -41,7 +41,7 @@ class TaskStatsInfoGetter;
class InternalTextLogsQueue;
struct ViewRuntimeData;
class QueryViewsLog;
class MemoryTrackerThreadSwitcher;
class ThreadGroupSwitcher;
using InternalTextLogsQueuePtr = std::shared_ptr<InternalTextLogsQueue>;
using InternalTextLogsQueueWeakPtr = std::weak_ptr<InternalTextLogsQueue>;
@ -106,6 +106,8 @@ public:
/// When new query starts, new thread group is created for it, current thread becomes master thread of the query
static ThreadGroupStatusPtr createForQuery(ContextPtr query_context_, FatalErrorCallback fatal_error_callback_ = {});
static ThreadGroupStatusPtr createForBackgroundProcess(ContextPtr storage_context);
std::vector<UInt64> getInvolvedThreadIds() const;
void linkThread(UInt64 thread_it);
@ -177,12 +179,6 @@ private:
bool performance_counters_finalized = false;
String query_id_from_query_context;
/// Requires access to query_id.
friend class MemoryTrackerThreadSwitcher;
void setQueryId(const String & query_id_)
{
query_id_from_query_context = query_id_;
}
struct TimePoint
{

14
src/Common/ZooKeeper/ZooKeeperImpl.cpp
View File

@ -669,8 +669,8 @@ void ZooKeeper::receiveThread()
earliest_operation = operations.begin()->second;
auto earliest_operation_deadline = earliest_operation->time + std::chrono::microseconds(args.operation_timeout_ms * 1000);
if (now > earliest_operation_deadline)
throw Exception(Error::ZOPERATIONTIMEOUT, "Operation timeout (deadline already expired) for path: {}",
earliest_operation->request->getPath());
throw Exception(Error::ZOPERATIONTIMEOUT, "Operation timeout (deadline of {} ms already expired) for path: {}",
args.operation_timeout_ms, earliest_operation->request->getPath());
max_wait_us = std::chrono::duration_cast<std::chrono::microseconds>(earliest_operation_deadline - now).count();
}
}
@ -687,12 +687,12 @@ void ZooKeeper::receiveThread()
{
if (earliest_operation)
{
throw Exception(Error::ZOPERATIONTIMEOUT, "Operation timeout (no response) for request {} for path: {}",
toString(earliest_operation->request->getOpNum()), earliest_operation->request->getPath());
throw Exception(Error::ZOPERATIONTIMEOUT, "Operation timeout (no response in {} ms) for request {} for path: {}",
args.operation_timeout_ms, toString(earliest_operation->request->getOpNum()), earliest_operation->request->getPath());
}
waited_us += max_wait_us;
if (waited_us >= args.session_timeout_ms * 1000)
throw Exception(Error::ZOPERATIONTIMEOUT, "Nothing is received in session timeout");
throw Exception(Error::ZOPERATIONTIMEOUT, "Nothing is received in session timeout of {} ms", args.session_timeout_ms);
}
@ -1080,7 +1080,7 @@ void ZooKeeper::pushRequest(RequestInfo && info)
if (requests_queue.isFinished())
throw Exception(Error::ZSESSIONEXPIRED, "Session expired");
throw Exception(Error::ZOPERATIONTIMEOUT, "Cannot push request to queue within operation timeout");
throw Exception(Error::ZOPERATIONTIMEOUT, "Cannot push request to queue within operation timeout of {} ms", args.operation_timeout_ms);
}
}
catch (...)
@ -1332,7 +1332,7 @@ void ZooKeeper::close()
request_info.request = std::make_shared<ZooKeeperCloseRequest>(std::move(request));
if (!requests_queue.tryPush(std::move(request_info), args.operation_timeout_ms))
throw Exception(Error::ZOPERATIONTIMEOUT, "Cannot push close request to queue within operation timeout");
throw Exception(Error::ZOPERATIONTIMEOUT, "Cannot push close request to queue within operation timeout of {} ms", args.operation_timeout_ms);
ProfileEvents::increment(ProfileEvents::ZooKeeperClose);
}

2
src/Core/Settings.h
View File

@ -131,7 +131,7 @@ class IColumn;
M(Bool, allow_suspicious_fixed_string_types, false, "In CREATE TABLE statement allows creating columns of type FixedString(n) with n > 256. FixedString with length >= 256 is suspicious and most likely indicates misusage", 0) \
M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
M(Bool, compile_aggregate_expressions, false, "Compile aggregate functions to native code. This feature has a bug and should not be used.", 0) \
M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
M(Bool, compile_sort_description, true, "Compile sort description to native code.", 0) \
M(UInt64, min_count_to_compile_sort_description, 3, "The number of identical sort descriptions before they are JIT-compiled", 0) \

2
src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
View File

@ -1184,7 +1184,7 @@ String CachedOnDiskReadBufferFromFile::getInfoForLog()
implementation_buffer_read_range_str = "None";
String current_file_segment_info;
if (current_file_segment_it == file_segments_holder->file_segments.end())
if (current_file_segment_it != file_segments_holder->file_segments.end())
current_file_segment_info = (*current_file_segment_it)->getInfoForLog();
else
current_file_segment_info = "None";

34
src/Interpreters/ActionsDAG.cpp
View File

@ -762,11 +762,10 @@ NameSet ActionsDAG::foldActionsByProjection(
}
ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const Node *, std::string> & new_inputs, const NodeRawConstPtrs & required_outputs)
ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const Node *, const Node *> & new_inputs, const NodeRawConstPtrs & required_outputs)
{
auto dag = std::make_unique<ActionsDAG>();
std::unordered_map<const Node *, size_t> new_input_to_pos;
std::unordered_map<const Node *, const Node *> inputs_mapping;
std::unordered_map<const Node *, const Node *> mapping;
struct Frame
{
@ -796,11 +795,21 @@ ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const
if (!node)
{
bool should_rename = !rename.empty() && new_input->result_name != rename;
const auto & input_name = should_rename ? rename : new_input->result_name;
node = &dag->addInput(input_name, new_input->result_type);
if (should_rename)
node = &dag->addAlias(*node, new_input->result_name);
/// It is possible to have a few aliases on the same column.
/// We may want to replace all the aliases,
/// in this case they should have a single input as a child.
auto & mapped_input = inputs_mapping[rename];
if (!mapped_input)
{
bool should_rename = new_input->result_name != rename->result_name;
const auto & input_name = should_rename ? rename->result_name : new_input->result_name;
mapped_input = &dag->addInput(input_name, new_input->result_type);
if (should_rename)
mapped_input = &dag->addAlias(*mapped_input, new_input->result_name);
}
node = mapped_input;
}
stack.pop_back();
@ -836,7 +845,14 @@ ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const
}
for (const auto * output : required_outputs)
dag->outputs.push_back(mapping[output]);
{
/// Keep the names for outputs.
/// Add an alias if the mapped node has a different result name.
const auto * mapped_output = mapping[output];
if (output->result_name != mapped_output->result_name)
mapped_output = &dag->addAlias(*mapped_output, output->result_name);
dag->outputs.push_back(mapped_output);
}
return dag;
}

10
src/Interpreters/ActionsDAG.h
View File

@ -221,9 +221,11 @@ public:
const String & predicate_column_name = {},
bool add_missing_keys = true);
/// Get an ActionsDAG where:
/// * Subtrees from new_inputs are converted to inputs with specified names.
/// * Outputs are taken from required_outputs.
/// Get an ActionsDAG in a following way:
/// * Traverse a tree starting from required_outputs
/// * If there is a node from new_inputs keys, replace it to INPUT
/// * INPUT name should be taken from new_inputs mapped node name
/// * Mapped nodes may be the same nodes, and in this case there would be a single INPUT
/// Here want to substitute some expressions to columns from projection.
/// This function expects that all required_outputs can be calculated from nodes in new_inputs.
/// If not, exception will happen.
@ -240,7 +242,7 @@ public:
/// \ /
/// c * d - e
static ActionsDAGPtr foldActionsByProjection(
const std::unordered_map<const Node *, std::string> & new_inputs,
const std::unordered_map<const Node *, const Node *> & new_inputs,
const NodeRawConstPtrs & required_outputs);
/// Reorder the output nodes using given position mapping.

10
src/Interpreters/Cache/FileSegment.cpp
View File

@ -201,7 +201,7 @@ void FileSegment::resetDownloadingStateUnlocked([[maybe_unused]] std::unique_loc
size_t current_downloaded_size = getDownloadedSizeUnlocked(segment_lock);
/// range().size() can equal 0 in case of write-though cache.
if (current_downloaded_size != 0 && current_downloaded_size == range().size())
if (!is_unbound && current_downloaded_size != 0 && current_downloaded_size == range().size())
setDownloadedUnlocked(segment_lock);
else
setDownloadState(State::PARTIALLY_DOWNLOADED);
@ -343,7 +343,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset)
ErrorCodes::LOGICAL_ERROR,
"Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size);
if (current_downloaded_size == range().size())
if (!is_unbound && current_downloaded_size == range().size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded");
if (!cache_writer)
@ -689,7 +689,8 @@ String FileSegment::getInfoForLogUnlocked(std::unique_lock<std::mutex> & segment
info << "first non-downloaded offset: " << getFirstNonDownloadedOffsetUnlocked(segment_lock) << ", ";
info << "caller id: " << getCallerId() << ", ";
info << "detached: " << is_detached << ", ";
info << "kind: " << toString(segment_kind);
info << "kind: " << toString(segment_kind) << ", ";
info << "unbound: " << is_unbound;
return info.str();
}
@ -785,6 +786,7 @@ FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std
snapshot->downloaded_size = file_segment->getDownloadedSizeUnlocked(segment_lock);
snapshot->download_state = file_segment->download_state;
snapshot->segment_kind = file_segment->getKind();
snapshot->is_unbound = file_segment->is_unbound;
return snapshot;
}
@ -905,6 +907,8 @@ String FileSegmentsHolder::toString()
if (!ranges.empty())
ranges += ", ";
ranges += file_segment->range().toString();
if (file_segment->is_unbound)
ranges += "(unbound)";
}
return ranges;
}

1
src/Interpreters/Cache/FileSegment.h
View File

@ -159,6 +159,7 @@ public:
FileSegmentKind getKind() const { return segment_kind; }
bool isPersistent() const { return segment_kind == FileSegmentKind::Persistent; }
bool isUnbound() const { return is_unbound; }
using UniqueId = std::pair<FileCacheKey, size_t>;
UniqueId getUniqueId() const { return std::pair(key(), offset()); }

13
src/Interpreters/Cache/WriteBufferToFileSegment.cpp
View File

@ -2,6 +2,8 @@
#include <Interpreters/Cache/FileSegment.h>
#include <IO/SwapHelper.h>
#include <base/scope_guard.h>
#include <Common/logger_useful.h>
namespace DB
@ -10,20 +12,23 @@ namespace DB
namespace ErrorCodes
{
extern const int NOT_ENOUGH_SPACE;
extern const int LOGICAL_ERROR;
}
WriteBufferToFileSegment::WriteBufferToFileSegment(FileSegment * file_segment_)
: WriteBufferFromFileDecorator(file_segment_->detachWriter()), file_segment(file_segment_)
{
auto downloader = file_segment->getOrSetDownloader();
if (downloader != FileSegment::getCallerId())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to set a downloader. ({})", file_segment->getInfoForLog());
}
/// If it throws an exception, the file segment will be incomplete, so you should not use it in the future.
void WriteBufferToFileSegment::nextImpl()
{
auto downloader [[maybe_unused]] = file_segment->getOrSetDownloader();
chassert(downloader == FileSegment::getCallerId());
SCOPE_EXIT({
file_segment->completePartAndResetDownloader();
});
size_t bytes_to_write = offset();
/// In case of an error, we don't need to finalize the file segment

233
src/Interpreters/Context.cpp
View File

@ -172,6 +172,8 @@ namespace ErrorCodes
extern const int UNKNOWN_READ_METHOD;
extern const int NOT_IMPLEMENTED;
extern const int UNKNOWN_FUNCTION;
extern const int ILLEGAL_COLUMN;
extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH;
}
@ -219,7 +221,11 @@ struct ContextSharedPart : boost::noncopyable
ConfigurationPtr config; /// Global configuration settings.
String tmp_path; /// Path to the temporary files that occur when processing the request.
TemporaryDataOnDiskScopePtr temp_data_on_disk; /// Temporary files that occur when processing the request accounted here.
/// All temporary files that occur when processing the requests accounted here.
/// Child scopes for more fine-grained accounting are created per user/query/etc.
/// Initialized once during server startup.
TemporaryDataOnDiskScopePtr root_temp_data_on_disk;
mutable std::unique_ptr<EmbeddedDictionaries> embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization.
mutable std::unique_ptr<ExternalDictionariesLoader> external_dictionaries_loader;
@ -752,25 +758,35 @@ Strings Context::getWarnings() const
}
/// TODO: remove, use `getTempDataOnDisk`
VolumePtr Context::getTemporaryVolume() const
VolumePtr Context::getGlobalTemporaryVolume() const
{
auto lock = getLock();
if (shared->temp_data_on_disk)
return shared->temp_data_on_disk->getVolume();
/// Calling this method we just bypass the `temp_data_on_disk` and write to the file on the volume directly.
/// Volume is the same for `root_temp_data_on_disk` (always set) and `temp_data_on_disk` (if it's set).
if (shared->root_temp_data_on_disk)
return shared->root_temp_data_on_disk->getVolume();
return nullptr;
}
TemporaryDataOnDiskScopePtr Context::getTempDataOnDisk() const
{
auto lock = getLock();
if (this->temp_data_on_disk)
return this->temp_data_on_disk;
return shared->temp_data_on_disk;
auto lock = getLock();
return shared->root_temp_data_on_disk;
}
TemporaryDataOnDiskScopePtr Context::getSharedTempDataOnDisk() const
{
auto lock = getLock();
return shared->root_temp_data_on_disk;
}
void Context::setTempDataOnDisk(TemporaryDataOnDiskScopePtr temp_data_on_disk_)
{
auto lock = getLock();
/// It's set from `ProcessList::insert` in `executeQueryImpl` before query execution
/// so no races with `getTempDataOnDisk` which is called from query execution.
this->temp_data_on_disk = std::move(temp_data_on_disk_);
}
@ -780,7 +796,7 @@ void Context::setPath(const String & path)
shared->path = path;
if (shared->tmp_path.empty() && !shared->temp_data_on_disk)
if (shared->tmp_path.empty() && !shared->root_temp_data_on_disk)
shared->tmp_path = shared->path + "tmp/";
if (shared->flags_path.empty())
@ -836,6 +852,11 @@ static VolumePtr createLocalSingleDiskVolume(const std::string & path)
void Context::setTemporaryStoragePath(const String & path, size_t max_size)
{
auto lock = getLock();
if (shared->root_temp_data_on_disk)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary storage is already set");
shared->tmp_path = path;
if (!shared->tmp_path.ends_with('/'))
shared->tmp_path += '/';
@ -847,17 +868,23 @@ void Context::setTemporaryStoragePath(const String & path, size_t max_size)
setupTmpPath(shared->log, disk->getPath());
}
shared->temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
}
void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_size)
{
std::lock_guard lock(shared->storage_policies_mutex);
StoragePolicyPtr tmp_policy;
{
/// lock in required only for accessing `shared->merge_tree_storage_policy_selector`
/// StoragePolicy itself is immutable.
std::lock_guard storage_policies_lock(shared->storage_policies_mutex);
tmp_policy = getStoragePolicySelector(storage_policies_lock)->get(policy_name);
}
StoragePolicyPtr tmp_policy = getStoragePolicySelector(lock)->get(policy_name);
if (tmp_policy->getVolumes().size() != 1)
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG,
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG,
"Policy '{}' is used temporary files, such policy should have exactly one volume", policy_name);
VolumePtr volume = tmp_policy->getVolume(0);
if (volume->getDisks().empty())
@ -882,12 +909,21 @@ void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_s
setupTmpPath(shared->log, disk->getPath());
}
shared->temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
}
auto lock = getLock();
if (shared->root_temp_data_on_disk)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary storage is already set");
shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, max_size);
}
void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t max_size)
{
auto lock = getLock();
if (shared->root_temp_data_on_disk)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary storage is already set");
auto disk_ptr = getDisk(cache_disk_name);
if (!disk_ptr)
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Disk '{}' is not found", cache_disk_name);
@ -904,7 +940,7 @@ void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t
shared->tmp_path = file_cache->getBasePath();
VolumePtr volume = createLocalSingleDiskVolume(shared->tmp_path);
shared->temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, file_cache.get(), max_size);
shared->root_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(volume, file_cache.get(), max_size);
}
void Context::setFlagsPath(const String & path)
@ -1361,6 +1397,22 @@ void Context::addQueryFactoriesInfo(QueryLogFactories factory_type, const String
}
}
static bool findIdentifier(const ASTFunction * function)
{
if (!function || !function->arguments)
return false;
if (const auto * arguments = function->arguments->as<ASTExpressionList>())
{
for (const auto & argument : arguments->children)
{
if (argument->as<ASTIdentifier>())
return true;
if (const auto * f = argument->as<ASTFunction>(); f && findIdentifier(f))
return true;
}
}
return false;
}
StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const ASTSelectQuery * select_query_hint)
{
@ -1407,62 +1459,125 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const
}
throw;
}
if (getSettingsRef().use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint() && hasInsertionTable())
uint64_t use_structure_from_insertion_table_in_table_functions = getSettingsRef().use_structure_from_insertion_table_in_table_functions;
if (use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint() && hasInsertionTable())
{
const auto & structure_hint = DatabaseCatalog::instance().getTable(getInsertionTable(), shared_from_this())->getInMemoryMetadataPtr()->getColumns();
const auto & insert_structure = DatabaseCatalog::instance().getTable(getInsertionTable(), shared_from_this())->getInMemoryMetadataPtr()->getColumns();
DB::ColumnsDescription structure_hint;
bool use_columns_from_insert_query = true;
/// use_structure_from_insertion_table_in_table_functions=2 means `auto`
if (select_query_hint && getSettingsRef().use_structure_from_insertion_table_in_table_functions == 2)
/// Insert table matches columns against SELECT expression by position, so we want to map
/// insert table columns to table function columns through names from SELECT expression.
auto insert_column = insert_structure.begin();
auto insert_structure_end = insert_structure.end(); /// end iterator of the range covered by possible asterisk
auto virtual_column_names = table_function_ptr->getVirtualsToCheckBeforeUsingStructureHint();
bool asterisk = false;
const auto & expression_list = select_query_hint->select()->as<ASTExpressionList>()->children;
const auto * expression = expression_list.begin();
/// We want to go through SELECT expression list and correspond each expression to column in insert table
/// which type will be used as a hint for the file structure inference.
for (; expression != expression_list.end() && insert_column != insert_structure_end; ++expression)
{
const auto * expression_list = select_query_hint->select()->as<ASTExpressionList>();
std::unordered_set<String> virtual_column_names = table_function_ptr->getVirtualsToCheckBeforeUsingStructureHint();
Names columns_names;
bool have_asterisk = false;
/// First, check if we have only identifiers, asterisk and literals in select expression,
/// and if no, we cannot use the structure from insertion table.
for (const auto & expression : expression_list->children)
if (auto * identifier = (*expression)->as<ASTIdentifier>())
{
if (auto * identifier = expression->as<ASTIdentifier>())
if (!virtual_column_names.contains(identifier->name()))
{
columns_names.push_back(identifier->name());
}
else if (expression->as<ASTAsterisk>())
{
have_asterisk = true;
}
else if (!expression->as<ASTLiteral>())
{
use_columns_from_insert_query = false;
break;
}
}
if (asterisk)
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Asterisk cannot be mixed with column list in INSERT SELECT query.");
/// Check that all identifiers are column names from insertion table and not virtual column names from storage.
for (const auto & column_name : columns_names)
{
if (!structure_hint.has(column_name) || virtual_column_names.contains(column_name))
{
use_columns_from_insert_query = false;
break;
}
}
use_columns_from_insert_query = false;
break;
}
/// If we don't have asterisk but only subset of columns, we should use
/// structure from insertion table only in case when table function
/// supports reading subset of columns from data.
if (use_columns_from_insert_query && !have_asterisk && !columns_names.empty())
{
/// For input function we should check if input format supports reading subset of columns.
if (table_function_ptr->getName() == "input")
use_columns_from_insert_query = FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(getInsertFormat());
structure_hint.add({ identifier->name(), insert_column->type });
}
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
use_columns_from_insert_query = table_function_ptr->supportsReadingSubsetOfColumns();
++insert_column;
}
else if ((*expression)->as<ASTAsterisk>())
{
if (asterisk)
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Only one asterisk can be used in INSERT SELECT query.");
use_columns_from_insert_query = false;
break;
}
if (!structure_hint.empty())
{
if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Asterisk cannot be mixed with column list in INSERT SELECT query.");
use_columns_from_insert_query = false;
break;
}
asterisk = true;
}
else if (auto * function = (*expression)->as<ASTFunction>())
{
if (use_structure_from_insertion_table_in_table_functions == 2 && findIdentifier(function))
{
use_columns_from_insert_query = false;
break;
}
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
++insert_column;
}
else
{
/// Once we hit asterisk we want to find end of the range covered by asterisk
/// contributing every further SELECT expression to the tail of insert structure
if (asterisk)
--insert_structure_end;
else
++insert_column;
}
}
if (use_structure_from_insertion_table_in_table_functions == 2 && !asterisk)
{
/// For input function we should check if input format supports reading subset of columns.
if (table_function_ptr->getName() == "input")
use_columns_from_insert_query = FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(getInsertFormat());
else
use_columns_from_insert_query = table_function_ptr->supportsReadingSubsetOfColumns();
}
if (use_columns_from_insert_query)
table_function_ptr->setStructureHint(structure_hint);
{
if (expression == expression_list.end())
{
/// Append tail of insert structure to the hint
if (asterisk)
{
for (; insert_column != insert_structure_end; ++insert_column)
structure_hint.add({ insert_column->name, insert_column->type });
}
if (!structure_hint.empty())
table_function_ptr->setStructureHint(structure_hint);
} else if (use_structure_from_insertion_table_in_table_functions == 1)
throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns in insert table less than required by SELECT expression.");
}
}
res = table_function_ptr->execute(table_expression, shared_from_this(), table_function_ptr->getName());
@ -3460,7 +3575,7 @@ void Context::shutdown()
}
/// Special volumes might also use disks that require shutdown.
auto & tmp_data = shared->temp_data_on_disk;
auto & tmp_data = shared->root_temp_data_on_disk;
if (tmp_data && tmp_data->getVolume())
{
auto & disks = tmp_data->getVolume()->getDisks();

3
src/Interpreters/Context.h
View File

@ -472,9 +472,10 @@ public:
/// A list of warnings about server configuration to place in `system.warnings` table.
Strings getWarnings() const;
VolumePtr getTemporaryVolume() const; /// TODO: remove, use `getTempDataOnDisk`
VolumePtr getGlobalTemporaryVolume() const; /// TODO: remove, use `getTempDataOnDisk`
TemporaryDataOnDiskScopePtr getTempDataOnDisk() const;
TemporaryDataOnDiskScopePtr getSharedTempDataOnDisk() const;
void setTempDataOnDisk(TemporaryDataOnDiskScopePtr temp_data_on_disk_);
void setPath(const String & path);

33
src/Interpreters/FillingRow.cpp
View File

@ -1,5 +1,6 @@
#include <Interpreters/FillingRow.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <IO/Operators.h>
namespace DB
@ -44,21 +45,27 @@ bool FillingRow::operator==(const FillingRow & other) const
return true;
}
bool FillingRow::operator>=(const FillingRow & other) const
{
return !(*this < other);
}
bool FillingRow::next(const FillingRow & to_row)
{
const size_t row_size = size();
size_t pos = 0;
/// Find position we need to increment for generating next row.
for (size_t s = size(); pos < s; ++pos)
for (; pos < row_size; ++pos)
if (!row[pos].isNull() && !to_row.row[pos].isNull() && !equals(row[pos], to_row.row[pos]))
break;
if (pos == size() || less(to_row.row[pos], row[pos], getDirection(pos)))
if (pos == row_size || less(to_row.row[pos], row[pos], getDirection(pos)))
return false;
/// If we have any 'fill_to' value at position greater than 'pos',
/// we need to generate rows up to 'fill_to' value.
for (size_t i = size() - 1; i > pos; --i)
for (size_t i = row_size - 1; i > pos; --i)
{
if (getFillDescription(i).fill_to.isNull() || row[i].isNull())
continue;
@ -84,7 +91,7 @@ bool FillingRow::next(const FillingRow & to_row)
{
bool is_less = false;
size_t i = pos + 1;
for (; i < size(); ++i)
for (; i < row_size; ++i)
{
const auto & fill_from = getFillDescription(i).fill_from;
if (!fill_from.isNull())
@ -107,4 +114,22 @@ void FillingRow::initFromDefaults(size_t from_pos)
row[i] = getFillDescription(i).fill_from;
}
String FillingRow::dump() const
{
WriteBufferFromOwnString out;
for (size_t i = 0; i < row.size(); ++i)
{
if (i != 0)
out << ", ";
out << row[i].dump();
}
return out.str();
}
WriteBuffer & operator<<(WriteBuffer & out, const FillingRow & row)
{
out << row.dump();
return out;
}
}

7
src/Interpreters/FillingRow.h
View File

@ -1,7 +1,5 @@
#pragma once
#include <Core/SortDescription.h>
#include <Core/InterpolateDescription.h>
#include <Columns/IColumn.h>
namespace DB
@ -30,13 +28,18 @@ public:
size_t size() const { return row.size(); }
bool operator<(const FillingRow & other) const;
bool operator==(const FillingRow & other) const;
bool operator>=(const FillingRow & other) const;
int getDirection(size_t index) const { return sort_description[index].direction; }
FillColumnDescription & getFillDescription(size_t index) { return sort_description[index].fill_description; }
String dump() const;
private:
Row row;
SortDescription sort_description;
};
WriteBuffer & operator<<(WriteBuffer & out, const FillingRow & row);
}

2
src/Interpreters/JoinedTables.cpp
View File

@ -308,7 +308,7 @@ std::shared_ptr<TableJoin> JoinedTables::makeTableJoin(const ASTSelectQuery & se
auto settings = context->getSettingsRef();
MultiEnum<JoinAlgorithm> join_algorithm = settings.join_algorithm;
auto table_join = std::make_shared<TableJoin>(settings, context->getTemporaryVolume());
auto table_join = std::make_shared<TableJoin>(settings, context->getGlobalTemporaryVolume());
const ASTTablesInSelectQueryElement * ast_join = select_query.join();
const auto & table_to_join = ast_join->table_expression->as<ASTTableExpression &>();

2
src/Interpreters/MergeJoin.cpp
View File

@ -1045,7 +1045,7 @@ std::shared_ptr<Block> MergeJoin::loadRightBlock(size_t pos) const
void MergeJoin::initRightTableWriter()
{
disk_writer = std::make_unique<SortedBlocksWriter>(size_limits, table_join->getTemporaryVolume(),
disk_writer = std::make_unique<SortedBlocksWriter>(size_limits, table_join->getGlobalTemporaryVolume(),
right_sample_block, right_sort_description, max_rows_in_right_block, max_files_to_merge,
table_join->temporaryFilesCodec());
disk_writer->addBlocks(right_blocks);

19
src/Interpreters/MutationsInterpreter.cpp
View File

@ -373,11 +373,11 @@ MutationsInterpreter::MutationsInterpreter(
ContextPtr context_,
bool can_execute_,
bool return_all_columns_,
bool return_deleted_rows_)
bool return_mutated_rows_)
: MutationsInterpreter(
Source(std::move(storage_)),
metadata_snapshot_, std::move(commands_), std::move(context_),
can_execute_, return_all_columns_, return_deleted_rows_)
can_execute_, return_all_columns_, return_mutated_rows_)
{
if (can_execute_ && dynamic_cast<const MergeTreeData *>(source.getStorage().get()))
{
@ -396,11 +396,11 @@ MutationsInterpreter::MutationsInterpreter(
ContextPtr context_,
bool can_execute_,
bool return_all_columns_,
bool return_deleted_rows_)
bool return_mutated_rows_)
: MutationsInterpreter(
Source(storage_, std::move(source_part_)),
metadata_snapshot_, std::move(commands_), std::move(context_),
can_execute_, return_all_columns_, return_deleted_rows_)
can_execute_, return_all_columns_, return_mutated_rows_)
{
}
@ -411,7 +411,7 @@ MutationsInterpreter::MutationsInterpreter(
ContextPtr context_,
bool can_execute_,
bool return_all_columns_,
bool return_deleted_rows_)
bool return_mutated_rows_)
: source(std::move(source_))
, metadata_snapshot(metadata_snapshot_)
, commands(std::move(commands_))
@ -419,7 +419,7 @@ MutationsInterpreter::MutationsInterpreter(
, can_execute(can_execute_)
, select_limits(SelectQueryOptions().analyze(!can_execute).ignoreLimits().ignoreProjections())
, return_all_columns(return_all_columns_)
, return_deleted_rows(return_deleted_rows_)
, return_mutated_rows(return_mutated_rows_)
{
prepare(!can_execute);
}
@ -600,7 +600,7 @@ void MutationsInterpreter::prepare(bool dry_run)
for (auto & command : commands)
{
// we can return deleted rows only if it's the only present command
assert(command.type == MutationCommand::DELETE || !return_deleted_rows);
assert(command.type == MutationCommand::DELETE || command.type == MutationCommand::UPDATE || !return_mutated_rows);
if (command.type == MutationCommand::DELETE)
{
@ -610,7 +610,7 @@ void MutationsInterpreter::prepare(bool dry_run)
auto predicate = getPartitionAndPredicateExpressionForMutationCommand(command);
if (!return_deleted_rows)
if (!return_mutated_rows)
predicate = makeASTFunction("isZeroOrNull", predicate);
stages.back().filters.push_back(predicate);
@ -697,6 +697,9 @@ void MutationsInterpreter::prepare(bool dry_run)
type_literal);
stages.back().column_to_updated.emplace(column, updated_column);
if (condition && return_mutated_rows)
stages.back().filters.push_back(condition);
}
if (!affected_materialized.empty())

10
src/Interpreters/MutationsInterpreter.h
View File

@ -48,7 +48,7 @@ public:
ContextPtr context_,
bool can_execute_,
bool return_all_columns_ = false,
bool return_deleted_rows_ = false);
bool return_mutated_rows_ = false);
/// Special case for MergeTree
MutationsInterpreter(
@ -59,7 +59,7 @@ public:
ContextPtr context_,
bool can_execute_,
bool return_all_columns_ = false,
bool return_deleted_rows_ = false);
bool return_mutated_rows_ = false);
void validate();
size_t evaluateCommandsSize();
@ -136,7 +136,7 @@ private:
ContextPtr context_,
bool can_execute_,
bool return_all_columns_,
bool return_deleted_rows_);
bool return_mutated_rows_);
void prepare(bool dry_run);
@ -210,8 +210,8 @@ private:
// whether all columns should be returned, not just updated
bool return_all_columns;
// whether we should return deleted or nondeleted rows on DELETE mutation
bool return_deleted_rows;
// whether we should return mutated or all existing rows
bool return_mutated_rows;
};
}

2
src/Interpreters/ProcessList.cpp
View File

@ -625,7 +625,7 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList *
if (global_context)
{
size_t size_limit = global_context->getSettingsRef().max_temporary_data_on_disk_size_for_user;
user_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(global_context->getTempDataOnDisk(), size_limit);
user_temp_data_on_disk = std::make_shared<TemporaryDataOnDiskScope>(global_context->getSharedTempDataOnDisk(), size_limit);
}
}

3
src/Interpreters/SystemLog.cpp
View File

@ -503,6 +503,9 @@ void SystemLog<LogElement>::prepareTable()
rename->elements.emplace_back(std::move(elem));
auto query_context = Context::createCopy(context);
/// As this operation is performed automatically we don't want it to fail because of user dependencies on log tables
query_context->setSetting("check_table_dependencies", Field{false});
query_context->setSetting("check_referential_table_dependencies", Field{false});
query_context->makeQueryContext();
InterpreterRenameQuery(rename, query_context).execute();

2
src/Interpreters/TableJoin.h
View File

@ -209,7 +209,7 @@ public:
JoinStrictness strictness() const { return table_join.strictness; }
bool sameStrictnessAndKind(JoinStrictness, JoinKind) const;
const SizeLimits & sizeLimits() const { return size_limits; }
VolumePtr getTemporaryVolume() { return tmp_volume; }
VolumePtr getGlobalTemporaryVolume() { return tmp_volume; }
bool isEnabledAlgorithm(JoinAlgorithm val) const
{

16
src/Interpreters/ThreadStatusExt.cpp
View File

@ -73,6 +73,22 @@ ThreadGroupStatusPtr ThreadGroupStatus::createForQuery(ContextPtr query_context_
return group;
}
ThreadGroupStatusPtr ThreadGroupStatus::createForBackgroundProcess(ContextPtr storage_context)
{
auto group = std::make_shared<ThreadGroupStatus>(storage_context);
group->memory_tracker.setDescription("background process to apply mutate/merge in table");
/// However settings from storage context have to be applied
const Settings & settings = storage_context->getSettingsRef();
group->memory_tracker.setProfilerStep(settings.memory_profiler_step);
group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator);
if (settings.memory_tracker_fault_probability > 0.0)
group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
return group;
}
void ThreadGroupStatus::attachQueryForLog(const String & query_, UInt64 normalized_hash)
{
auto hash = normalized_hash ? normalized_hash : normalizedQueryHash<false>(query_);

25
src/Interpreters/tests/gtest_lru_file_cache.cpp
View File

@ -562,7 +562,7 @@ TEST_F(FileCacheTest, writeBuffer)
DB::FileCache cache(cache_base_path, settings);
cache.initialize();
auto write_to_cache = [&cache](const String & key, const Strings & data)
auto write_to_cache = [&cache](const String & key, const Strings & data, bool flush)
{
CreateFileSegmentSettings segment_settings;
segment_settings.kind = FileSegmentKind::Temporary;
@ -572,14 +572,31 @@ TEST_F(FileCacheTest, writeBuffer)
EXPECT_EQ(holder.file_segments.size(), 1);
auto & segment = holder.file_segments.front();
WriteBufferToFileSegment out(segment.get());
std::list<std::thread> threads;
std::mutex mu;
for (const auto & s : data)
out.write(s.data(), s.size());
{
/// Write from diffetent threads to check
/// that no assertions inside cache related to downloaderId are triggered
threads.emplace_back([&]
{
std::unique_lock lock(mu);
out.write(s.data(), s.size());
/// test different buffering scenarios
if (flush)
{
out.next();
}
});
}
for (auto & t : threads)
t.join();
return holder;
};
std::vector<fs::path> file_segment_paths;
{
auto holder = write_to_cache("key1", {"abc", "defg"});
auto holder = write_to_cache("key1", {"abc", "defg"}, false);
file_segment_paths.emplace_back(holder.file_segments.front()->getPathInLocalCache());
ASSERT_EQ(fs::file_size(file_segment_paths.back()), 7);
@ -587,7 +604,7 @@ TEST_F(FileCacheTest, writeBuffer)
ASSERT_EQ(cache.getUsedCacheSize(), 7);
{
auto holder2 = write_to_cache("key2", {"1", "22", "333", "4444", "55555"});
auto holder2 = write_to_cache("key2", {"1", "22", "333", "4444", "55555"}, true);
file_segment_paths.emplace_back(holder2.file_segments.front()->getPathInLocalCache());
ASSERT_EQ(fs::file_size(file_segment_paths.back()), 15);

3
src/Parsers/IAST.h
View File

@ -208,7 +208,8 @@ public:
FormatSettings(WriteBuffer & ostr_, const FormatSettings & other)
: ostr(ostr_), hilite(other.hilite), one_line(other.one_line),
always_quote_identifiers(other.always_quote_identifiers), identifier_quoting_style(other.identifier_quoting_style)
always_quote_identifiers(other.always_quote_identifiers), identifier_quoting_style(other.identifier_quoting_style),
show_secrets(other.show_secrets)
{
nl_or_ws = one_line ? ' ' : '\n';
}

2
src/Planner/PlannerJoinTree.cpp
View File

@ -994,7 +994,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
}
}
auto table_join = std::make_shared<TableJoin>(settings, query_context->getTemporaryVolume());
auto table_join = std::make_shared<TableJoin>(settings, query_context->getGlobalTemporaryVolume());
table_join->getTableJoin() = join_node.toASTTableJoin()->as<ASTTableJoin &>();
table_join->getTableJoin().kind = join_kind;

181
src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
View File

@ -44,7 +44,6 @@
M(arrow::Type::UINT8, DB::UInt8) \
M(arrow::Type::INT8, DB::Int8) \
M(arrow::Type::INT16, DB::Int16) \
M(arrow::Type::INT32, DB::Int32) \
M(arrow::Type::UINT64, DB::UInt64) \
M(arrow::Type::INT64, DB::Int64) \
M(arrow::Type::DURATION, DB::Int64) \
@ -165,6 +164,73 @@ static ColumnWithTypeAndName readColumnWithFixedStringData(std::shared_ptr<arrow
return {std::move(internal_column), std::move(internal_type), column_name};
}
template <typename ValueType>
static ColumnWithTypeAndName readColumnWithBigIntegerFromFixedBinaryData(std::shared_ptr<arrow::ChunkedArray> & arrow_column, const String & column_name, const DataTypePtr & column_type)
{
const auto * fixed_type = assert_cast<arrow::FixedSizeBinaryType *>(arrow_column->type().get());
size_t fixed_len = fixed_type->byte_width();
if (fixed_len != sizeof(ValueType))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Cannot insert data into {} column from fixed size binary, expected data with size {}, got {}",
column_type->getName(),
sizeof(ValueType),
fixed_len);
auto internal_column = column_type->createColumn();
auto & data = assert_cast<ColumnVector<ValueType> &>(*internal_column).getData();
data.reserve(arrow_column->length());
for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i)
{
arrow::FixedSizeBinaryArray & chunk = dynamic_cast<arrow::FixedSizeBinaryArray &>(*(arrow_column->chunk(chunk_i)));
const auto * raw_data = reinterpret_cast<const ValueType *>(chunk.raw_values());
data.insert_assume_reserved(raw_data, raw_data + chunk.length());
}
return {std::move(internal_column), column_type, column_name};
}
template <typename ColumnType, typename ValueType = typename ColumnType::ValueType>
static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData(std::shared_ptr<arrow::ChunkedArray> & arrow_column, const String & column_name, const DataTypePtr & column_type)
{
size_t total_size = 0;
for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i)
{
auto & chunk = dynamic_cast<arrow::BinaryArray &>(*(arrow_column->chunk(chunk_i)));
const size_t chunk_length = chunk.length();
for (size_t i = 0; i != chunk_length; ++i)
{
if (!chunk.IsNull(i) && chunk.value_length(i) != sizeof(ValueType))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Cannot insert data into {} column from binary value, expected data with size {}, got {}",
column_type->getName(),
sizeof(ValueType),
chunk.value_length(i));
total_size += chunk_length;
}
}
auto internal_column = column_type->createColumn();
auto & integer_column = assert_cast<ColumnType &>(*internal_column);
integer_column.reserve(total_size);
for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i)
{
auto & chunk = dynamic_cast<arrow::BinaryArray &>(*(arrow_column->chunk(chunk_i)));
for (size_t value_i = 0, length = static_cast<size_t>(chunk.length()); value_i < length; ++value_i)
{
if (chunk.IsNull(value_i))
integer_column.insertDefault();
else
integer_column.insertData(chunk.Value(value_i).data(), chunk.Value(value_i).size());
}
}
return {std::move(internal_column), column_type, column_name};
}
static ColumnWithTypeAndName readColumnWithBooleanData(std::shared_ptr<arrow::ChunkedArray> & arrow_column, const String & column_name)
{
auto internal_type = DataTypeFactory::instance().get("Bool");
@ -537,7 +603,7 @@ static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(std::shared_ptr<arrow:
for (size_t i = 0; i != chunk_length; ++i)
{
/// If at least one value size is not 16 bytes, fallback to reading String column and further cast to IPv6.
if (chunk.value_length(i) != sizeof(IPv6))
if (!chunk.IsNull(i) && chunk.value_length(i) != sizeof(IPv6))
return readColumnWithStringData<arrow::BinaryArray>(arrow_column, column_name);
}
total_size += chunk_length;
@ -545,14 +611,40 @@ static ColumnWithTypeAndName readIPv6ColumnFromBinaryData(std::shared_ptr<arrow:
auto internal_type = std::make_shared<DataTypeIPv6>();
auto internal_column = internal_type->createColumn();
auto & data = assert_cast<ColumnIPv6 &>(*internal_column).getData();
data.reserve(total_size * sizeof(IPv6));
auto & ipv6_column = assert_cast<ColumnIPv6 &>(*internal_column);
ipv6_column.reserve(total_size);
for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i)
{
auto & chunk = dynamic_cast<arrow::BinaryArray &>(*(arrow_column->chunk(chunk_i)));
const auto * raw_data = reinterpret_cast<const IPv6 *>(chunk.raw_data() + chunk.raw_value_offsets()[0]);
data.insert_assume_reserved(raw_data, raw_data + chunk.length());
for (size_t value_i = 0, length = static_cast<size_t>(chunk.length()); value_i < length; ++value_i)
{
if (chunk.IsNull(value_i))
ipv6_column.insertDefault();
else
ipv6_column.insertData(chunk.Value(value_i).data(), chunk.Value(value_i).size());
}
}
return {std::move(internal_column), std::move(internal_type), column_name};
}
static ColumnWithTypeAndName readIPv4ColumnWithInt32Data(std::shared_ptr<arrow::ChunkedArray> & arrow_column, const String & column_name)
{
auto internal_type = std::make_shared<DataTypeIPv4>();
auto internal_column = internal_type->createColumn();
auto & column_data = assert_cast<ColumnIPv4 &>(*internal_column).getData();
column_data.reserve(arrow_column->length());
for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i)
{
std::shared_ptr<arrow::Array> chunk = arrow_column->chunk(chunk_i);
if (chunk->length() == 0)
continue;
/// buffers[0] is a null bitmap and buffers[1] are actual values
std::shared_ptr<arrow::Buffer> buffer = chunk->data()->buffers[1];
const auto * raw_data = reinterpret_cast<const IPv4 *>(buffer->data()) + chunk->offset();
column_data.insert_assume_reserved(raw_data, raw_data + chunk->length());
}
return {std::move(internal_column), std::move(internal_type), column_name};
}
@ -566,7 +658,8 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
bool allow_null_type,
bool skip_columns_with_unsupported_types,
bool & skipped,
DataTypePtr type_hint = nullptr)
DataTypePtr type_hint = nullptr,
bool is_map_nested = false)
{
if (!is_nullable && (arrow_column->null_count() || (type_hint && type_hint->isNullable())) && arrow_column->type()->id() != arrow::Type::LIST
&& arrow_column->type()->id() != arrow::Type::MAP && arrow_column->type()->id() != arrow::Type::STRUCT &&
@ -589,12 +682,49 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
case arrow::Type::STRING:
case arrow::Type::BINARY:
{
if (type_hint && isIPv6(type_hint))
return readIPv6ColumnFromBinaryData(arrow_column, column_name);
if (type_hint)
{
switch (type_hint->getTypeId())
{
case TypeIndex::IPv6:
return readIPv6ColumnFromBinaryData(arrow_column, column_name);
/// ORC format outputs big integers as binary column, because there is no fixed binary in ORC.
case TypeIndex::Int128:
return readColumnWithBigNumberFromBinaryData<ColumnInt128>(arrow_column, column_name, type_hint);
case TypeIndex::UInt128:
return readColumnWithBigNumberFromBinaryData<ColumnUInt128>(arrow_column, column_name, type_hint);
case TypeIndex::Int256:
return readColumnWithBigNumberFromBinaryData<ColumnInt256>(arrow_column, column_name, type_hint);
case TypeIndex::UInt256:
return readColumnWithBigNumberFromBinaryData<ColumnUInt256>(arrow_column, column_name, type_hint);
/// ORC doesn't support Decimal256 as separate type. We read and write it as binary data.
case TypeIndex::Decimal256:
return readColumnWithBigNumberFromBinaryData<ColumnDecimal<Decimal256>>(arrow_column, column_name, type_hint);
default:;
}
}
return readColumnWithStringData<arrow::BinaryArray>(arrow_column, column_name);
}
case arrow::Type::FIXED_SIZE_BINARY:
{
if (type_hint)
{
switch (type_hint->getTypeId())
{
case TypeIndex::Int128:
return readColumnWithBigIntegerFromFixedBinaryData<Int128>(arrow_column, column_name, type_hint);
case TypeIndex::UInt128:
return readColumnWithBigIntegerFromFixedBinaryData<UInt128>(arrow_column, column_name, type_hint);
case TypeIndex::Int256:
return readColumnWithBigIntegerFromFixedBinaryData<Int256>(arrow_column, column_name, type_hint);
case TypeIndex::UInt256:
return readColumnWithBigIntegerFromFixedBinaryData<UInt256>(arrow_column, column_name, type_hint);
default:;
}
}
return readColumnWithFixedStringData(arrow_column, column_name);
}
case arrow::Type::LARGE_BINARY:
case arrow::Type::LARGE_STRING:
return readColumnWithStringData<arrow::LargeBinaryArray>(arrow_column, column_name);
@ -621,6 +751,14 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
column.type = std::make_shared<DataTypeDateTime>();
return column;
}
case arrow::Type::INT32:
{
/// ORC format doesn't have unsigned integers and we output IPv4 as Int32.
/// We should allow to read it back from Int32.
if (type_hint && isIPv4(type_hint))
return readIPv4ColumnWithInt32Data(arrow_column, column_name);
return readColumnWithNumericData<Int32>(arrow_column, column_name);
}
case arrow::Type::TIMESTAMP:
return readColumnWithTimestampData(arrow_column, column_name);
case arrow::Type::DECIMAL128:
@ -630,14 +768,18 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
case arrow::Type::MAP:
{
DataTypePtr nested_type_hint;
DataTypePtr key_type_hint;
if (type_hint)
{
const auto * map_type_hint = typeid_cast<const DataTypeMap *>(type_hint.get());
if (map_type_hint)
{
nested_type_hint = assert_cast<const DataTypeArray *>(map_type_hint->getNestedType().get())->getNestedType();
key_type_hint = map_type_hint->getKeyType();
}
}
auto arrow_nested_column = getNestedArrowColumn(arrow_column);
auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, nested_type_hint);
auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, nested_type_hint, true);
if (skipped)
return {};
@ -645,8 +787,21 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
const auto * tuple_column = assert_cast<const ColumnTuple *>(nested_column.column.get());
const auto * tuple_type = assert_cast<const DataTypeTuple *>(nested_column.type.get());
auto map_column = ColumnMap::create(tuple_column->getColumnPtr(0), tuple_column->getColumnPtr(1), offsets_column);
auto map_type = std::make_shared<DataTypeMap>(tuple_type->getElements()[0], tuple_type->getElements()[1]);
auto key_column = tuple_column->getColumnPtr(0);
auto key_type = tuple_type->getElements()[0];
auto value_column = tuple_column->getColumnPtr(1);
auto value_type = tuple_type->getElements()[1];
if (key_type_hint && !key_type_hint->equals(*key_type))
{
/// Cast key column to target type, because it can happen
/// that parsed type cannot be ClickHouse Map key type.
key_column = castColumn({key_column, key_type, "key"}, key_type_hint);
key_type = key_type_hint;
}
auto map_column = ColumnMap::create(key_column, value_column, offsets_column);
auto map_type = std::make_shared<DataTypeMap>(key_type, value_type);
return {std::move(map_column), std::move(map_type), column_name};
}
case arrow::Type::LIST:
@ -690,7 +845,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn(
DataTypePtr nested_type_hint;
if (tuple_type_hint)
{
if (tuple_type_hint->haveExplicitNames())
if (tuple_type_hint->haveExplicitNames() && !is_map_nested)
{
auto pos = tuple_type_hint->tryGetPositionByName(field_name);
if (pos)

311
src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp
View File

@ -26,7 +26,6 @@
#include <arrow/util/decimal.h>
#define FOR_INTERNAL_NUMERIC_TYPES(M) \
M(UInt8, arrow::UInt8Builder) \
M(Int8, arrow::Int8Builder) \
M(UInt16, arrow::UInt16Builder) \
M(Int16, arrow::Int16Builder) \
@ -65,8 +64,10 @@ namespace DB
{
{"UInt8", arrow::uint8()},
{"Int8", arrow::int8()},
{"Enum8", arrow::int8()},
{"UInt16", arrow::uint16()},
{"Int16", arrow::int16()},
{"Enum16", arrow::int16()},
{"UInt32", arrow::uint32()},
{"Int32", arrow::int32()},
{"UInt64", arrow::uint64()},
@ -80,6 +81,11 @@ namespace DB
{"String", arrow::binary()},
{"FixedString", arrow::binary()},
{"Int128", arrow::fixed_size_binary(sizeof(Int128))},
{"UInt128", arrow::fixed_size_binary(sizeof(UInt128))},
{"Int256", arrow::fixed_size_binary(sizeof(Int256))},
{"UInt256", arrow::fixed_size_binary(sizeof(UInt256))},
};
@ -148,7 +154,7 @@ namespace DB
}
static void fillArrowArrayWithDateTime64ColumnData(
const DataTypeDateTime64 * type,
const DataTypePtr & type,
ColumnPtr write_column,
const PaddedPODArray<UInt8> * null_bytemap,
const String & format_name,
@ -156,11 +162,12 @@ namespace DB
size_t start,
size_t end)
{
const auto * datetime64_type = assert_cast<const DataTypeDateTime64 *>(type.get());
const auto & column = assert_cast<const ColumnDecimal<DateTime64> &>(*write_column);
arrow::TimestampBuilder & builder = assert_cast<arrow::TimestampBuilder &>(*array_builder);
arrow::Status status;
auto scale = type->getScale();
auto scale = datetime64_type->getScale();
bool need_rescale = scale % 3;
auto rescale_multiplier = DecimalUtils::scaleMultiplier<DateTime64::NativeType>(3 - scale % 3);
for (size_t value_i = start; value_i < end; ++value_i)
@ -186,7 +193,7 @@ namespace DB
static void fillArrowArray(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
@ -200,7 +207,7 @@ namespace DB
static void fillArrowArrayWithArrayColumnData(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
@ -231,7 +238,7 @@ namespace DB
static void fillArrowArrayWithTupleColumnData(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
@ -303,7 +310,7 @@ namespace DB
static void fillArrowArrayWithLowCardinalityColumnDataImpl(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> *,
arrow::ArrayBuilder * array_builder,
String format_name,
@ -359,7 +366,7 @@ namespace DB
static void fillArrowArrayWithLowCardinalityColumnData(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
@ -424,13 +431,12 @@ namespace DB
const auto & internal_data = internal_column.getChars();
size_t fixed_length = internal_column.getN();
arrow::FixedSizeBinaryBuilder & builder = assert_cast<arrow::FixedSizeBinaryBuilder &>(*array_builder);
arrow::Status status;
PaddedPODArray<UInt8> arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end);
const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data();
const uint8_t * data_start = reinterpret_cast<const uint8_t *>(internal_data.data() + start * fixed_length);
status = builder.AppendValues(data_start, end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
arrow::Status status = builder.AppendValues(data_start, end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
checkStatus(status, write_column->getName(), format_name);
}
@ -446,13 +452,12 @@ namespace DB
const auto & internal_data = internal_column.getData();
size_t fixed_length = sizeof(IPv6);
arrow::FixedSizeBinaryBuilder & builder = assert_cast<arrow::FixedSizeBinaryBuilder &>(*array_builder);
arrow::Status status;
PaddedPODArray<UInt8> arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end);
const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data();
const uint8_t * data_start = reinterpret_cast<const uint8_t *>(internal_data.data()) + start * fixed_length;
status = builder.AppendValues(data_start, end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
arrow::Status status = builder.AppendValues(data_start, end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
checkStatus(status, write_column->getName(), format_name);
}
@ -466,11 +471,10 @@ namespace DB
{
const auto & internal_data = assert_cast<const ColumnIPv4 &>(*write_column).getData();
auto & builder = assert_cast<arrow::UInt32Builder &>(*array_builder);
arrow::Status status;
PaddedPODArray<UInt8> arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end);
const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data();
status = builder.AppendValues(&(internal_data.data() + start)->toUnderType(), end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
arrow::Status status = builder.AppendValues(&(internal_data.data() + start)->toUnderType(), end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
checkStatus(status, write_column->getName(), format_name);
}
@ -541,134 +545,6 @@ namespace DB
}
}
static void fillArrowArray(
const String & column_name,
ColumnPtr & column,
const std::shared_ptr<const IDataType> & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
size_t start,
size_t end,
bool output_string_as_string,
bool output_fixed_string_as_fixed_byte_array,
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
{
const String column_type_name = column_type->getFamilyName();
if (column_type->isNullable())
{
const ColumnNullable * column_nullable = assert_cast<const ColumnNullable *>(column.get());
ColumnPtr nested_column = column_nullable->getNestedColumnPtr();
DataTypePtr nested_type = assert_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
const ColumnPtr & null_column = column_nullable->getNullMapColumnPtr();
const PaddedPODArray<UInt8> & bytemap = assert_cast<const ColumnVector<UInt8> &>(*null_column).getData();
fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
}
else if (isString(column_type))
{
if (output_string_as_string)
fillArrowArrayWithStringColumnData<ColumnString, arrow::StringBuilder>(column, null_bytemap, format_name, array_builder, start, end);
else
fillArrowArrayWithStringColumnData<ColumnString, arrow::BinaryBuilder>(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isFixedString(column_type))
{
if (output_fixed_string_as_fixed_byte_array)
fillArrowArrayWithFixedStringColumnData(column, null_bytemap, format_name, array_builder, start, end);
else if (output_string_as_string)
fillArrowArrayWithStringColumnData<ColumnFixedString, arrow::StringBuilder>(column, null_bytemap, format_name, array_builder, start, end);
else
fillArrowArrayWithStringColumnData<ColumnFixedString, arrow::BinaryBuilder>(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isIPv6(column_type))
{
fillArrowArrayWithIPv6ColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isIPv4(column_type))
{
fillArrowArrayWithIPv4ColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isDate(column_type))
{
fillArrowArrayWithDateColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isDateTime(column_type))
{
fillArrowArrayWithDateTimeColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isDate32(column_type))
{
fillArrowArrayWithDate32ColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
else if (isArray(column_type))
{
fillArrowArrayWithArrayColumnData<arrow::ListBuilder>(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
}
else if (isTuple(column_type))
{
fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
}
else if (column_type->getTypeId() == TypeIndex::LowCardinality)
{
fillArrowArrayWithLowCardinalityColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
}
else if (isMap(column_type))
{
ColumnPtr column_array = assert_cast<const ColumnMap *>(column.get())->getNestedColumnPtr();
DataTypePtr array_type = assert_cast<const DataTypeMap *>(column_type.get())->getNestedType();
fillArrowArrayWithArrayColumnData<arrow::MapBuilder>(column_name, column_array, array_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
}
else if (isDecimal(column_type))
{
auto fill_decimal = [&](const auto & types) -> bool
{
using Types = std::decay_t<decltype(types)>;
using ToDataType = typename Types::LeftType;
if constexpr (
std::is_same_v<ToDataType,DataTypeDecimal<Decimal32>>
|| std::is_same_v<ToDataType, DataTypeDecimal<Decimal64>>
|| std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>)
{
fillArrowArrayWithDecimalColumnData<ToDataType, Int128, arrow::Decimal128, arrow::Decimal128Builder>(column, null_bytemap, array_builder, format_name, start, end);
return true;
}
if constexpr (std::is_same_v<ToDataType,DataTypeDecimal<Decimal256>>)
{
fillArrowArrayWithDecimalColumnData<ToDataType, Int256, arrow::Decimal256, arrow::Decimal256Builder>(column, null_bytemap, array_builder, format_name, start, end);
return true;
}
return false;
};
if (!callOnIndexAndDataType<void>(column_type->getTypeId(), fill_decimal))
throw Exception{ErrorCodes::LOGICAL_ERROR, "Cannot fill arrow array with decimal data with type {}", column_type_name};
}
else if (isDateTime64(column_type))
{
const auto * datetime64_type = assert_cast<const DataTypeDateTime64 *>(column_type.get());
fillArrowArrayWithDateTime64ColumnData(datetime64_type, column, null_bytemap, format_name, array_builder, start, end);
}
else if (isBool(column_type))
{
fillArrowArrayWithBoolColumnData(column, null_bytemap, format_name, array_builder, start, end);
}
#define DISPATCH(CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE) \
else if (#CPP_NUMERIC_TYPE == column_type_name) \
{ \
fillArrowArrayWithNumericColumnData<CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE>(column, null_bytemap, format_name, array_builder, start, end); \
}
FOR_INTERNAL_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
else
{
throw Exception(ErrorCodes::UNKNOWN_TYPE,
"Internal type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type_name, column_name, format_name);
}
}
template <typename DataType, typename FieldType, typename ArrowDecimalType, typename ArrowBuilder>
static void fillArrowArrayWithDecimalColumnData(
ColumnPtr write_column,
@ -697,6 +573,157 @@ namespace DB
checkStatus(status, write_column->getName(), format_name);
}
template <typename ColumnType>
static void fillArrowArrayWithBigIntegerColumnData(
ColumnPtr write_column,
const PaddedPODArray<UInt8> * null_bytemap,
const String & format_name,
arrow::ArrayBuilder* array_builder,
size_t start,
size_t end)
{
const auto & internal_column = assert_cast<const ColumnType &>(*write_column);
const auto & internal_data = internal_column.getData();
size_t fixed_length = sizeof(typename ColumnType::ValueType);
arrow::FixedSizeBinaryBuilder & builder = assert_cast<arrow::FixedSizeBinaryBuilder &>(*array_builder);
PaddedPODArray<UInt8> arrow_null_bytemap = revertNullByteMap(null_bytemap, start, end);
const UInt8 * arrow_null_bytemap_raw_ptr = arrow_null_bytemap.empty() ? nullptr : arrow_null_bytemap.data();
const uint8_t * data_start = reinterpret_cast<const uint8_t *>(internal_data.data()) + start * fixed_length;
arrow::Status status = builder.AppendValues(data_start, end - start, reinterpret_cast<const uint8_t *>(arrow_null_bytemap_raw_ptr));
checkStatus(status, write_column->getName(), format_name);
}
static void fillArrowArray(
const String & column_name,
ColumnPtr & column,
const DataTypePtr & column_type,
const PaddedPODArray<UInt8> * null_bytemap,
arrow::ArrayBuilder * array_builder,
String format_name,
size_t start,
size_t end,
bool output_string_as_string,
bool output_fixed_string_as_fixed_byte_array,
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
{
const String column_type_name = column_type->getFamilyName();
WhichDataType which(column_type);
switch (column_type->getTypeId())
{
case TypeIndex::Nullable:
{
const ColumnNullable * column_nullable = assert_cast<const ColumnNullable *>(column.get());
ColumnPtr nested_column = column_nullable->getNestedColumnPtr();
DataTypePtr nested_type = assert_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
const ColumnPtr & null_column = column_nullable->getNullMapColumnPtr();
const PaddedPODArray<UInt8> & bytemap = assert_cast<const ColumnVector<UInt8> &>(*null_column).getData();
fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
break;
}
case TypeIndex::String:
{
if (output_string_as_string)
fillArrowArrayWithStringColumnData<ColumnString, arrow::StringBuilder>(column, null_bytemap, format_name, array_builder, start, end);
else
fillArrowArrayWithStringColumnData<ColumnString, arrow::BinaryBuilder>(column, null_bytemap, format_name, array_builder, start, end);
break;
}
case TypeIndex::FixedString:
{
if (output_fixed_string_as_fixed_byte_array)
fillArrowArrayWithFixedStringColumnData(column, null_bytemap, format_name, array_builder, start, end);
else if (output_string_as_string)
fillArrowArrayWithStringColumnData<ColumnFixedString, arrow::StringBuilder>(column, null_bytemap, format_name, array_builder, start, end);
else
fillArrowArrayWithStringColumnData<ColumnFixedString, arrow::BinaryBuilder>(column, null_bytemap, format_name, array_builder, start, end);
break;
}
case TypeIndex::IPv6:
fillArrowArrayWithIPv6ColumnData(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::IPv4:
fillArrowArrayWithIPv4ColumnData(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Date:
fillArrowArrayWithDateColumnData(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::DateTime:
fillArrowArrayWithDateTimeColumnData(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Date32:
fillArrowArrayWithDate32ColumnData(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Array:
fillArrowArrayWithArrayColumnData<arrow::ListBuilder>(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
break;
case TypeIndex::Tuple:
fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
break;
case TypeIndex::LowCardinality:
fillArrowArrayWithLowCardinalityColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
break;
case TypeIndex::Map:
{
ColumnPtr column_array = assert_cast<const ColumnMap *>(column.get())->getNestedColumnPtr();
DataTypePtr array_type = assert_cast<const DataTypeMap *>(column_type.get())->getNestedType();
fillArrowArrayWithArrayColumnData<arrow::MapBuilder>(column_name, column_array, array_type, null_bytemap, array_builder, format_name, start, end, output_string_as_string, output_fixed_string_as_fixed_byte_array, dictionary_values);
break;
}
case TypeIndex::Decimal32:
fillArrowArrayWithDecimalColumnData<DataTypeDecimal32, Int128, arrow::Decimal128, arrow::Decimal128Builder>(column, null_bytemap, array_builder, format_name, start, end);
break;
case TypeIndex::Decimal64:
fillArrowArrayWithDecimalColumnData<DataTypeDecimal64, Int128, arrow::Decimal128, arrow::Decimal128Builder>(column, null_bytemap, array_builder, format_name, start, end);
break;
case TypeIndex::Decimal128:
fillArrowArrayWithDecimalColumnData<DataTypeDecimal128, Int128, arrow::Decimal128, arrow::Decimal128Builder>(column, null_bytemap, array_builder, format_name, start, end);
break;
case TypeIndex::Decimal256:
fillArrowArrayWithDecimalColumnData<DataTypeDecimal256, Int256, arrow::Decimal256, arrow::Decimal256Builder>(column, null_bytemap, array_builder, format_name, start, end);
break;
case TypeIndex::DateTime64:
fillArrowArrayWithDateTime64ColumnData(column_type, column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::UInt8:
{
if (isBool(column_type))
fillArrowArrayWithBoolColumnData(column, null_bytemap, format_name, array_builder, start, end);
else
fillArrowArrayWithNumericColumnData<UInt8, arrow::UInt8Builder>(column, null_bytemap, format_name, array_builder, start, end);
break;
}
case TypeIndex::Enum8:
fillArrowArrayWithNumericColumnData<Int8, arrow::Int8Builder>(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Enum16:
fillArrowArrayWithNumericColumnData<Int16, arrow::Int16Builder>(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Int128:
fillArrowArrayWithBigIntegerColumnData<ColumnInt128>(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::UInt128:
fillArrowArrayWithBigIntegerColumnData<ColumnUInt128>(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::Int256:
fillArrowArrayWithBigIntegerColumnData<ColumnInt256>(column, null_bytemap, format_name, array_builder, start, end);
break;
case TypeIndex::UInt256:
fillArrowArrayWithBigIntegerColumnData<ColumnUInt256>(column, null_bytemap, format_name, array_builder, start, end);
break;
#define DISPATCH(CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE) \
case TypeIndex::CPP_NUMERIC_TYPE: \
fillArrowArrayWithNumericColumnData<CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE>(column, null_bytemap, format_name, array_builder, start, end); \
break;
FOR_INTERNAL_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
default:
throw Exception(ErrorCodes::UNKNOWN_TYPE, "Internal type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type_name, column_name, format_name);
}
}
static std::shared_ptr<arrow::DataType> getArrowTypeForLowCardinalityIndexes(ColumnPtr indexes_column)
{
/// Arrow docs recommend preferring signed integers over unsigned integers for representing dictionary indices.

35
src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp
View File

@ -93,10 +93,12 @@ std::unique_ptr<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr &
return orc::createPrimitiveType(orc::TypeKind::BOOLEAN);
return orc::createPrimitiveType(orc::TypeKind::BYTE);
}
case TypeIndex::Enum8: [[fallthrough]];
case TypeIndex::Int8:
{
return orc::createPrimitiveType(orc::TypeKind::BYTE);
}
case TypeIndex::Enum16: [[fallthrough]];
case TypeIndex::UInt16: [[fallthrough]];
case TypeIndex::Int16:
{
@ -131,6 +133,12 @@ std::unique_ptr<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr &
{
return orc::createPrimitiveType(orc::TypeKind::TIMESTAMP);
}
case TypeIndex::Int128: [[fallthrough]];
case TypeIndex::UInt128: [[fallthrough]];
case TypeIndex::Int256: [[fallthrough]];
case TypeIndex::UInt256: [[fallthrough]];
case TypeIndex::Decimal256:
return orc::createPrimitiveType(orc::TypeKind::BINARY);
case TypeIndex::FixedString: [[fallthrough]];
case TypeIndex::String:
{
@ -309,6 +317,7 @@ void ORCBlockOutputFormat::writeColumn(
switch (type->getTypeId())
{
case TypeIndex::Enum8: [[fallthrough]];
case TypeIndex::Int8:
{
/// Note: Explicit cast to avoid clang-tidy error: 'signed char' to 'long' conversion; consider casting to 'unsigned char' first.
@ -320,6 +329,7 @@ void ORCBlockOutputFormat::writeColumn(
writeNumbers<UInt8, orc::LongVectorBatch>(orc_column, column, null_bytemap, [](const UInt8 & value){ return value; });
break;
}
case TypeIndex::Enum16: [[fallthrough]];
case TypeIndex::Int16:
{
writeNumbers<Int16, orc::LongVectorBatch>(orc_column, column, null_bytemap, [](const Int16 & value){ return value; });
@ -357,6 +367,26 @@ void ORCBlockOutputFormat::writeColumn(
writeNumbers<UInt64,orc::LongVectorBatch>(orc_column, column, null_bytemap, [](const UInt64 & value){ return value; });
break;
}
case TypeIndex::Int128:
{
writeStrings<ColumnInt128>(orc_column, column, null_bytemap);
break;
}
case TypeIndex::UInt128:
{
writeStrings<ColumnUInt128>(orc_column, column, null_bytemap);
break;
}
case TypeIndex::Int256:
{
writeStrings<ColumnInt256>(orc_column, column, null_bytemap);
break;
}
case TypeIndex::UInt256:
{
writeStrings<ColumnUInt256>(orc_column, column, null_bytemap);
break;
}
case TypeIndex::Float32:
{
writeNumbers<Float32, orc::DoubleVectorBatch>(orc_column, column, null_bytemap, [](const Float32 & value){ return value; });
@ -432,6 +462,11 @@ void ORCBlockOutputFormat::writeColumn(
[](Int128 value){ return orc::Int128(value >> 64, (value << 64) >> 64); });
break;
}
case TypeIndex::Decimal256:
{
writeStrings<ColumnDecimal<Decimal256>>(orc_column, column, null_bytemap);
break;
}
case TypeIndex::Nullable:
{
const auto & nullable_column = assert_cast<const ColumnNullable &>(column);

76
src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
View File

@ -104,16 +104,25 @@ static bool hasNullableOrMissingColumn(const DAGIndex & index, const Names & nam
return false;
}
struct AggregateFunctionMatch
{
const AggregateDescription * description = nullptr;
DataTypes argument_types;
};
using AggregateFunctionMatches = std::vector<AggregateFunctionMatch>;
/// Here we try to match aggregate functions from the query to
/// aggregate functions from projection.
bool areAggregatesMatch(
std::optional<AggregateFunctionMatches> matchAggregateFunctions(
const AggregateProjectionInfo & info,
const AggregateDescriptions & aggregates,
const MatchedTrees::Matches & matches,
const DAGIndex & query_index,
const DAGIndex & proj_index)
{
AggregateFunctionMatches res;
/// Index (projection agg function name) -> pos
std::unordered_map<std::string, std::vector<size_t>> projection_aggregate_functions;
for (size_t i = 0; i < info.aggregates.size(); ++i)
@ -130,14 +139,20 @@ bool areAggregatesMatch(
// "Cannot match agg func {} by name {}",
// aggregate.column_name, aggregate.function->getName());
return false;
return {};
}
size_t num_args = aggregate.argument_names.size();
DataTypes argument_types;
argument_types.reserve(num_args);
auto & candidates = it->second;
bool found_match = false;
for (size_t idx : candidates)
{
argument_types.clear();
const auto & candidate = info.aggregates[idx];
/// Note: this check is a bit strict.
@ -148,9 +163,9 @@ bool areAggregatesMatch(
/// and we can't replace one to another from projection.
if (!candidate.function->getStateType()->equals(*aggregate.function->getStateType()))
{
LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Cannot match agg func {} vs {} by state {} vs {}",
aggregate.column_name, candidate.column_name,
candidate.function->getStateType()->getName(), aggregate.function->getStateType()->getName());
// LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Cannot match agg func {} vs {} by state {} vs {}",
// aggregate.column_name, candidate.column_name,
// candidate.function->getStateType()->getName(), aggregate.function->getStateType()->getName());
continue;
}
@ -166,6 +181,7 @@ bool areAggregatesMatch(
{
/// we can ignore arguments for count()
found_match = true;
res.push_back({&candidate, DataTypes()});
break;
}
}
@ -173,7 +189,6 @@ bool areAggregatesMatch(
/// Now, function names and types matched.
/// Next, match arguments from DAGs.
size_t num_args = aggregate.argument_names.size();
if (num_args != candidate.argument_names.size())
continue;
@ -215,6 +230,7 @@ bool areAggregatesMatch(
break;
}
argument_types.push_back(query_node->result_type);
++next_arg;
}
@ -222,14 +238,44 @@ bool areAggregatesMatch(
continue;
found_match = true;
res.push_back({&candidate, std::move(argument_types)});
break;
}
if (!found_match)
return false;
return {};
}
return true;
return res;
}
static void appendAggregateFunctions(
ActionsDAG & proj_dag,
const AggregateDescriptions & aggregates,
const AggregateFunctionMatches & matched_aggregates)
{
std::unordered_map<const AggregateDescription *, const ActionsDAG::Node *> inputs;
/// Just add all the aggregates to dag inputs.
auto & proj_dag_outputs = proj_dag.getOutputs();
size_t num_aggregates = aggregates.size();
for (size_t i = 0; i < num_aggregates; ++i)
{
const auto & aggregate = aggregates[i];
const auto & match = matched_aggregates[i];
auto type = std::make_shared<DataTypeAggregateFunction>(aggregate.function, match.argument_types, aggregate.parameters);
auto & input = inputs[match.description];
if (!input)
input = &proj_dag.addInput(match.description->column_name, std::move(type));
const auto * node = input;
if (node->result_name != aggregate.column_name)
node = &proj_dag.addAlias(*node, aggregate.column_name);
proj_dag_outputs.push_back(node);
}
}
ActionsDAGPtr analyzeAggregateProjection(
@ -250,7 +296,8 @@ ActionsDAGPtr analyzeAggregateProjection(
// static_cast<const void *>(match.node), (match.node ? match.node->result_name : ""), match.monotonicity != std::nullopt);
// }
if (!areAggregatesMatch(info, aggregates, matches, query_index, proj_index))
auto matched_aggregates = matchAggregateFunctions(info, aggregates, matches, query_index, proj_index);
if (!matched_aggregates)
return {};
ActionsDAG::NodeRawConstPtrs query_key_nodes;
@ -299,7 +346,7 @@ ActionsDAGPtr analyzeAggregateProjection(
std::stack<Frame> stack;
std::unordered_set<const ActionsDAG::Node *> visited;
std::unordered_map<const ActionsDAG::Node *, std::string> new_inputs;
std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> new_inputs;
for (const auto * key_node : query_key_nodes)
{
@ -321,7 +368,7 @@ ActionsDAGPtr analyzeAggregateProjection(
if (match.node && !match.monotonicity && proj_key_nodes.contains(match.node))
{
visited.insert(frame.node);
new_inputs[frame.node] = match.node->result_name;
new_inputs[frame.node] = match.node;
stack.pop();
continue;
}
@ -351,12 +398,7 @@ ActionsDAGPtr analyzeAggregateProjection(
// LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Folding actions by projection");
auto proj_dag = query.dag->foldActionsByProjection(new_inputs, query_key_nodes);
/// Just add all the aggregates to dag inputs.
auto & proj_dag_outputs = proj_dag->getOutputs();
for (const auto & aggregate : aggregates)
proj_dag_outputs.push_back(&proj_dag->addInput(aggregate.column_name, aggregate.function->getResultType()));
appendAggregateFunctions(*proj_dag, aggregates, *matched_aggregates);
return proj_dag;
}

183
src/Processors/Transforms/FillingTransform.cpp
View File

@ -8,11 +8,29 @@
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include <Common/FieldVisitorSum.h>
#include <Common/FieldVisitorToString.h>
#include <Common/logger_useful.h>
namespace DB
{
constexpr bool debug_logging_enabled = false;
template <typename T>
void logDebug(String key, const T & value, const char * separator = " : ")
{
if constexpr (debug_logging_enabled)
{
WriteBufferFromOwnString ss;
if constexpr (std::is_pointer_v<T>)
ss << *value;
else
ss << value;
LOG_DEBUG(&Poco::Logger::get("FillingTransform"), "{}{}{}", key, separator, ss.str());
}
}
namespace ErrorCodes
{
extern const int INVALID_WITH_FILL_EXPRESSION;
@ -233,27 +251,24 @@ FillingTransform::FillingTransform(
interpolate_column_positions.push_back(header_.getPositionByName(name));
}
/// prepare() is overrididen to call transform() after all chunks are processed
/// it can be necessary for suffix generation in case of WITH FILL .. TO is provided
IProcessor::Status FillingTransform::prepare()
{
if (input.isFinished() && !output.isFinished() && !has_input && !generate_suffix)
if (input.isFinished() && !output.isFinished() && !has_input && !all_chunks_processed)
{
should_insert_first = next_row < filling_row || first;
logDebug("prepare()", "all chunks processed");
all_chunks_processed = true;
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
next_row[i] = filling_row.getFillDescription(i).fill_to;
if (first || filling_row < next_row)
/// push output data to output port if we can
if (has_output && output.canPush())
{
/// Output if has data.
if (has_output)
{
output.pushData(std::move(output_data));
has_output = false;
}
generate_suffix = true;
return Status::Ready;
output.pushData(std::move(output_data));
has_output = false;
}
/// return Ready to call transform() for generating filling rows after latest chunk was processed
return Status::Ready;
}
return ISimpleTransform::prepare();
@ -316,8 +331,10 @@ static void insertFromFillingRow(const MutableColumnRawPtrs & filling_columns, c
interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0);
}
else
{
for (auto * interpolate_column : interpolate_columns)
interpolate_column->insertDefault();
}
for (auto * other_column : other_columns)
other_column->insertDefault();
@ -368,14 +385,75 @@ void FillingTransform::initColumns(
initColumnsByPositions(non_const_columns, input_other_columns, output_columns, output_other_columns, other_column_positions);
}
bool FillingTransform::generateSuffixIfNeeded(const Columns & input_columns, MutableColumns & result_columns)
{
logDebug("generateSuffixIfNeeded() filling_row", filling_row);
logDebug("generateSuffixIfNeeded() next_row", next_row);
logDebug("generateSuffixIfNeeded() first", first);
/// Determines should we insert filling row before start generating next rows.
bool should_insert_first = next_row < filling_row || first;
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
next_row[i] = filling_row.getFillDescription(i).fill_to;
logDebug("generateSuffixIfNeeded() next_row updated", next_row);
if (!first && filling_row >= next_row)
{
logDebug("generateSuffixIfNeeded()", "no need to generate suffix");
return false;
}
Columns input_fill_columns;
Columns input_interpolate_columns;
Columns input_other_columns;
MutableColumnRawPtrs res_fill_columns;
MutableColumnRawPtrs res_interpolate_columns;
MutableColumnRawPtrs res_other_columns;
initColumns(
input_columns,
input_fill_columns,
input_interpolate_columns,
input_other_columns,
result_columns,
res_fill_columns,
res_interpolate_columns,
res_other_columns);
if (first)
filling_row.initFromDefaults();
Block interpolate_block;
if (should_insert_first && filling_row < next_row)
{
interpolate(result_columns, interpolate_block);
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
}
while (filling_row.next(next_row))
{
interpolate(result_columns, interpolate_block);
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
}
return true;
}
void FillingTransform::transform(Chunk & chunk)
{
if (!chunk.hasRows() && !generate_suffix)
logDebug("new chunk rows", chunk.getNumRows());
logDebug("all chunks processed", all_chunks_processed);
/// if got chunk with no rows and it's not for suffix generation, then just skip it
/// Note: ExpressionTransform can return chunk with no rows, see 02579_fill_empty_chunk.sql for example
if (!chunk.hasRows() && !all_chunks_processed)
return;
Columns old_fill_columns;
Columns old_interpolate_columns;
Columns old_other_columns;
Columns input_fill_columns;
Columns input_interpolate_columns;
Columns input_other_columns;
MutableColumnRawPtrs res_fill_columns;
MutableColumnRawPtrs res_interpolate_columns;
MutableColumnRawPtrs res_other_columns;
@ -383,47 +461,31 @@ void FillingTransform::transform(Chunk & chunk)
Block interpolate_block;
if (generate_suffix)
if (all_chunks_processed)
{
const auto & empty_columns = input.getHeader().getColumns();
initColumns(
empty_columns,
old_fill_columns,
old_interpolate_columns,
old_other_columns,
result_columns,
res_fill_columns,
res_interpolate_columns,
res_other_columns);
chassert(!chunk.hasRows());
if (first)
filling_row.initFromDefaults();
if (should_insert_first && filling_row < next_row)
/// if all chunks are processed, then we may need to generate suffix for the following cases:
/// (1) when all data are processed and WITH FILL .. TO is provided
/// (2) for empty result set when WITH FILL FROM .. TO is provided (see PR #30888)
if (generateSuffixIfNeeded(input.getHeader().getColumns(), result_columns))
{
interpolate(result_columns, interpolate_block);
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
size_t num_output_rows = result_columns[0]->size();
chunk.setColumns(std::move(result_columns), num_output_rows);
}
interpolate(result_columns, interpolate_block);
while (filling_row.next(next_row))
{
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
interpolate(result_columns, interpolate_block);
}
size_t num_output_rows = result_columns[0]->size();
chunk.setColumns(std::move(result_columns), num_output_rows);
return;
}
chassert(chunk.hasRows());
const size_t num_rows = chunk.getNumRows();
auto old_columns = chunk.detachColumns();
auto input_columns = chunk.detachColumns();
initColumns(
old_columns,
old_fill_columns,
old_interpolate_columns,
old_other_columns,
input_columns,
input_fill_columns,
input_interpolate_columns,
input_other_columns,
result_columns,
res_fill_columns,
res_interpolate_columns,
@ -433,7 +495,7 @@ void FillingTransform::transform(Chunk & chunk)
{
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
{
auto current_value = (*old_fill_columns[i])[0];
auto current_value = (*input_fill_columns[i])[0];
const auto & fill_from = filling_row.getFillDescription(i).fill_from;
if (!fill_from.isNull() && !equals(current_value, fill_from))
@ -453,11 +515,16 @@ void FillingTransform::transform(Chunk & chunk)
for (size_t row_ind = 0; row_ind < num_rows; ++row_ind)
{
should_insert_first = next_row < filling_row;
logDebug("row", row_ind);
logDebug("filling_row", filling_row);
logDebug("next_row", next_row);
bool should_insert_first = next_row < filling_row;
logDebug("should_insert_first", should_insert_first);
for (size_t i = 0, size = filling_row.size(); i < size; ++i)
{
auto current_value = (*old_fill_columns[i])[row_ind];
auto current_value = (*input_fill_columns[i])[row_ind];
const auto & fill_to = filling_row.getFillDescription(i).fill_to;
if (fill_to.isNull() || less(current_value, fill_to, filling_row.getDirection(i)))
@ -465,6 +532,7 @@ void FillingTransform::transform(Chunk & chunk)
else
next_row[i] = fill_to;
}
logDebug("next_row updated", next_row);
/// A case, when at previous step row was initialized from defaults 'fill_from' values
/// and probably we need to insert it to block.
@ -474,16 +542,15 @@ void FillingTransform::transform(Chunk & chunk)
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
}
interpolate(result_columns, interpolate_block);
while (filling_row.next(next_row))
{
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
interpolate(result_columns, interpolate_block);
insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block);
}
copyRowFromColumns(res_fill_columns, old_fill_columns, row_ind);
copyRowFromColumns(res_interpolate_columns, old_interpolate_columns, row_ind);
copyRowFromColumns(res_other_columns, old_other_columns, row_ind);
copyRowFromColumns(res_fill_columns, input_fill_columns, row_ind);
copyRowFromColumns(res_interpolate_columns, input_interpolate_columns, row_ind);
copyRowFromColumns(res_other_columns, input_other_columns, row_ind);
}
saveLastRow(result_columns);

18
src/Processors/Transforms/FillingTransform.h
View File

@ -1,9 +1,9 @@
#pragma once
#include <Processors/ISimpleTransform.h>
#include <Core/SortDescription.h>
#include <Core/InterpolateDescription.h>
#include <Core/SortDescription.h>
#include <Interpreters/FillingRow.h>
#include <Processors/ISimpleTransform.h>
namespace DB
@ -29,7 +29,7 @@ protected:
private:
void saveLastRow(const MutableColumns & result_columns);
void interpolate(const MutableColumns& result_columns, Block & interpolate_block);
void interpolate(const MutableColumns & result_columns, Block & interpolate_block);
using MutableColumnRawPtrs = std::vector<IColumn *>;
void initColumns(
@ -42,6 +42,10 @@ private:
MutableColumnRawPtrs & output_interpolate_columns,
MutableColumnRawPtrs & output_other_columns);
bool generateSuffixIfNeeded(
const Columns & input_columns,
MutableColumns & result_columns);
const SortDescription sort_description; /// Contains only columns with WITH FILL.
const InterpolateDescriptionPtr interpolate_description; /// Contains INTERPOLATE columns
@ -54,13 +58,9 @@ private:
Positions other_column_positions;
std::vector<std::pair<size_t, NameAndTypePair>> input_positions; /// positions in result columns required for actions
ExpressionActionsPtr interpolate_actions;
bool first = true;
bool generate_suffix = false;
Columns last_row;
/// Determines should we insert filling row before start generating next rows.
bool should_insert_first = false;
bool first = true; /// flag to determine if transform is/will be called for the first time
bool all_chunks_processed = false; /// flag to determine if we have already processed all chunks
};
class FillingNoopTransform : public ISimpleTransform

2
src/Server/HTTPHandler.cpp
View File

@ -623,7 +623,7 @@ void HTTPHandler::processQuery(
if (buffer_until_eof)
{
const std::string tmp_path(server.context()->getTemporaryVolume()->getDisk()->getPath());
const std::string tmp_path(server.context()->getGlobalTemporaryVolume()->getDisk()->getPath());
const std::string tmp_path_template(fs::path(tmp_path) / "http_buffers/");
auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &)

7
src/Storages/MergeTree/MergeFromLogEntryTask.cpp
View File

@ -260,12 +260,15 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
auto table_id = storage.getStorageID();
task_context = Context::createCopy(storage.getContext());
task_context->makeQueryContext();
task_context->setCurrentQueryId("");
/// Add merge to list
const Settings & settings = storage.getContext()->getSettingsRef();
merge_mutate_entry = storage.getContext()->getMergeList().insert(
storage.getStorageID(),
future_merged_part,
settings);
task_context);
transaction_ptr = std::make_unique<MergeTreeData::Transaction>(storage, NO_TRANSACTION_RAW);
stopwatch_ptr = std::make_unique<Stopwatch>();

82
src/Storages/MergeTree/MergeList.cpp
View File

@ -11,56 +11,37 @@ namespace DB
{
MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_list_entry_)
: merge_list_entry(merge_list_entry_)
ThreadGroupSwitcher::ThreadGroupSwitcher(ThreadGroupStatusPtr thread_group)
{
// Each merge is executed into separate background processing pool thread
background_thread_memory_tracker = CurrentThread::getMemoryTracker();
background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent();
background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker);
chassert(thread_group);
prev_untracked_memory_limit = current_thread->untracked_memory_limit;
current_thread->untracked_memory_limit = merge_list_entry->max_untracked_memory;
/// might be nullptr
prev_thread_group = CurrentThread::getGroup();
/// Avoid accounting memory from another mutation/merge
/// (NOTE: consider moving such code to ThreadFromGlobalPool and related places)
prev_untracked_memory = current_thread->untracked_memory;
current_thread->untracked_memory = merge_list_entry->untracked_memory;
prev_query_id = std::string(current_thread->getQueryId());
current_thread->setQueryId(merge_list_entry->query_id);
CurrentThread::detachFromGroupIfNotDetached();
CurrentThread::attachToGroup(thread_group);
}
MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher()
ThreadGroupSwitcher::~ThreadGroupSwitcher()
{
// Unplug memory_tracker from current background processing pool thread
background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent);
current_thread->untracked_memory_limit = prev_untracked_memory_limit;
merge_list_entry->untracked_memory = current_thread->untracked_memory;
current_thread->untracked_memory = prev_untracked_memory;
current_thread->setQueryId(prev_query_id);
CurrentThread::detachFromGroupIfNotDetached();
if (prev_thread_group)
CurrentThread::attachToGroup(prev_thread_group);
}
MergeListElement::MergeListElement(
const StorageID & table_id_,
FutureMergedMutatedPartPtr future_part,
const Settings & settings)
const ContextPtr & context)
: table_id{table_id_}
, partition_id{future_part->part_info.partition_id}
, result_part_name{future_part->name}
, result_part_path{future_part->path}
, result_part_info{future_part->part_info}
, num_parts{future_part->parts.size()}
, max_untracked_memory(settings.max_untracked_memory)
, query_id(table_id.getShortName() + "::" + result_part_name)
, thread_id{getThreadId()}
, merge_type{future_part->merge_type}
, merge_algorithm{MergeAlgorithm::Undecided}
, description{"to apply mutate/merge in " + query_id}
{
for (const auto & source_part : future_part->parts)
{
@ -78,34 +59,7 @@ MergeListElement::MergeListElement(
is_mutation = (result_part_info.getDataVersion() != source_data_version);
}
memory_tracker.setDescription(description.c_str());
/// MemoryTracker settings should be set here, because
/// later (see MemoryTrackerThreadSwitcher)
/// parent memory tracker will be changed, and if merge executed from the
/// query (OPTIMIZE TABLE), all settings will be lost (since
/// current_thread::memory_tracker will have Thread level MemoryTracker,
/// which does not have any settings itself, it relies on the settings of the
/// thread_group::memory_tracker, but MemoryTrackerThreadSwitcher will reset parent).
memory_tracker.setProfilerStep(settings.memory_profiler_step);
memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator);
if (settings.memory_tracker_fault_probability > 0.0)
memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
/// Let's try to copy memory related settings from the query,
/// since settings that we have here is not from query, but global, from the table.
///
/// NOTE: Remember, that Thread level MemoryTracker does not have any settings,
/// so it's parent is required.
MemoryTracker * query_memory_tracker = CurrentThread::getMemoryTracker();
MemoryTracker * parent_query_memory_tracker;
if (query_memory_tracker->level == VariableContext::Thread &&
(parent_query_memory_tracker = query_memory_tracker->getParent()) &&
parent_query_memory_tracker != &total_memory_tracker)
{
memory_tracker.setOrRaiseHardLimit(parent_query_memory_tracker->getHardLimit());
}
thread_group = ThreadGroupStatus::createForBackgroundProcess(context);
}
MergeInfo MergeListElement::getInfo() const
@ -128,7 +82,7 @@ MergeInfo MergeListElement::getInfo() const
res.rows_read = rows_read.load(std::memory_order_relaxed);
res.rows_written = rows_written.load(std::memory_order_relaxed);
res.columns_written = columns_written.load(std::memory_order_relaxed);
res.memory_usage = memory_tracker.get();
res.memory_usage = getMemoryTracker().get();
res.thread_id = thread_id;
res.merge_type = toString(merge_type);
res.merge_algorithm = toString(merge_algorithm.load(std::memory_order_relaxed));
@ -142,14 +96,4 @@ MergeInfo MergeListElement::getInfo() const
return res;
}
MergeListElement::~MergeListElement()
{
if (untracked_memory != 0)
{
CurrentThread::getMemoryTracker()->adjustWithUntrackedMemory(untracked_memory);
untracked_memory = 0;
}
}
}

39
src/Storages/MergeTree/MergeList.h
View File

@ -5,6 +5,7 @@
#include <Common/Stopwatch.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <Common/ThreadStatus.h>
#include <Storages/MergeTree/MergeType.h>
#include <Storages/MergeTree/MergeAlgorithm.h>
#include <Storages/MergeTree/MergeTreePartInfo.h>
@ -63,23 +64,17 @@ struct Settings;
/**
* Since merge is executed with multiple threads, this class
* switches the parent MemoryTracker to account all the memory used.
* switches the parent MemoryTracker as part of the thread group to account all the memory used.
*/
class MemoryTrackerThreadSwitcher : boost::noncopyable
class ThreadGroupSwitcher : private boost::noncopyable
{
public:
explicit MemoryTrackerThreadSwitcher(MergeListEntry & merge_list_entry_);
~MemoryTrackerThreadSwitcher();
private:
MergeListEntry & merge_list_entry;
MemoryTracker * background_thread_memory_tracker;
MemoryTracker * background_thread_memory_tracker_prev_parent = nullptr;
Int64 prev_untracked_memory_limit;
Int64 prev_untracked_memory;
String prev_query_id;
};
explicit ThreadGroupSwitcher(ThreadGroupStatusPtr thread_group);
~ThreadGroupSwitcher();
using MemoryTrackerThreadSwitcherPtr = std::unique_ptr<MemoryTrackerThreadSwitcher>;
private:
ThreadGroupStatusPtr prev_thread_group;
};
struct MergeListElement : boost::noncopyable
{
@ -113,33 +108,23 @@ struct MergeListElement : boost::noncopyable
/// Updated only for Vertical algorithm
std::atomic<UInt64> columns_written{};
/// Used to adjust ThreadStatus::untracked_memory_limit
UInt64 max_untracked_memory;
/// Used to avoid losing any allocation context
UInt64 untracked_memory = 0;
/// Used for identifying mutations/merges in trace_log
std::string query_id;
UInt64 thread_id;
MergeType merge_type;
/// Detected after merge already started
std::atomic<MergeAlgorithm> merge_algorithm;
/// Description used for logging
/// Needs to outlive memory_tracker since it's used in its destructor
const String description{"Mutate/Merge"};
MemoryTracker memory_tracker{VariableContext::Process};
ThreadGroupStatusPtr thread_group;
MergeListElement(
const StorageID & table_id_,
FutureMergedMutatedPartPtr future_part,
const Settings & settings);
const ContextPtr & context);
MergeInfo getInfo() const;
MergeListElement * ptr() { return this; }
const MemoryTracker & getMemoryTracker() const { return thread_group->memory_tracker; }
~MergeListElement();
MergeListElement * ptr() { return this; }
MergeListElement & ref() { return *this; }
};

41
src/Storages/MergeTree/MergePlainMergeTreeTask.cpp
View File

@ -4,6 +4,8 @@
#include <Storages/StorageMergeTree.h>
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
#include <Common/ProfileEventsScope.h>
#include <Common/ProfileEvents.h>
namespace DB
{
@ -28,13 +30,17 @@ void MergePlainMergeTreeTask::onCompleted()
bool MergePlainMergeTreeTask::executeStep()
{
/// Metrics will be saved in the thread_group.
/// All metrics will be saved in the thread_group, including all scheduled tasks.
/// In profile_counters only metrics from this thread will be saved.
ProfileEventsScope profile_events_scope(&profile_counters);
/// Make out memory tracker a parent of current thread memory tracker
MemoryTrackerThreadSwitcherPtr switcher;
std::optional<ThreadGroupSwitcher> switcher;
if (merge_list_entry)
switcher = std::make_unique<MemoryTrackerThreadSwitcher>(*merge_list_entry);
{
switcher.emplace((*merge_list_entry)->thread_group);
}
switch (state)
{
@ -81,11 +87,11 @@ void MergePlainMergeTreeTask::prepare()
future_part = merge_mutate_entry->future_part;
stopwatch_ptr = std::make_unique<Stopwatch>();
const Settings & settings = storage.getContext()->getSettingsRef();
task_context = createTaskContext();
merge_list_entry = storage.getContext()->getMergeList().insert(
storage.getStorageID(),
future_part,
settings);
task_context);
write_part_log = [this] (const ExecutionStatus & execution_status)
{
@ -102,6 +108,19 @@ void MergePlainMergeTreeTask::prepare()
std::move(profile_counters_snapshot));
};
transfer_profile_counters_to_initial_query = [this, query_thread_group = CurrentThread::getGroup()] ()
{
if (query_thread_group)
{
auto task_thread_group = (*merge_list_entry)->thread_group;
auto task_counters_snapshot = task_thread_group->performance_counters.getPartiallyAtomicSnapshot();
auto & query_counters = query_thread_group->performance_counters;
for (ProfileEvents::Event i = ProfileEvents::Event(0); i < ProfileEvents::end(); ++i)
query_counters.incrementNoTrace(i, task_counters_snapshot[i]);
}
};
merge_task = storage.merger_mutator.mergePartsToTemporaryPart(
future_part,
metadata_snapshot,
@ -109,7 +128,7 @@ void MergePlainMergeTreeTask::prepare()
{} /* projection_merge_list_element */,
table_lock_holder,
time(nullptr),
storage.getContext(),
task_context,
merge_mutate_entry->tagger->reserved_space,
deduplicate,
deduplicate_by_columns,
@ -129,6 +148,16 @@ void MergePlainMergeTreeTask::finish()
write_part_log({});
storage.incrementMergedPartsProfileEvent(new_part->getType());
transfer_profile_counters_to_initial_query();
}
ContextMutablePtr MergePlainMergeTreeTask::createTaskContext() const
{
auto context = Context::createCopy(storage.getContext());
context->makeQueryContext();
auto queryId = storage.getStorageID().getShortName() + "::" + future_part->name;
context->setCurrentQueryId(queryId);
return context;
}
}

5
src/Storages/MergeTree/MergePlainMergeTreeTask.h
View File

@ -80,6 +80,7 @@ private:
UInt64 priority{0};
std::function<void(const ExecutionStatus &)> write_part_log;
std::function<void()> transfer_profile_counters_to_initial_query;
IExecutableTask::TaskResultCallback task_result_callback;
MergeTaskPtr merge_task{nullptr};
@ -87,6 +88,10 @@ private:
MergeTreeTransactionPtr txn;
ProfileEvents::Counters profile_counters;
ContextMutablePtr task_context;
ContextMutablePtr createTaskContext() const;
};

6
src/Storages/MergeTree/MergeTask.cpp
View File

@ -273,7 +273,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare()
ctx->compression_codec = global_ctx->data->getCompressionCodecForPart(
global_ctx->merge_list_element_ptr->total_size_bytes_compressed, global_ctx->new_data_part->ttl_infos, global_ctx->time_of_merge);
ctx->tmp_disk = global_ctx->context->getTemporaryVolume()->getDisk();
ctx->tmp_disk = global_ctx->context->getGlobalTemporaryVolume()->getDisk();
switch (global_ctx->chosen_merge_algorithm)
{
@ -702,13 +702,11 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c
if (projection.type == ProjectionDescription::Type::Aggregate)
projection_merging_params.mode = MergeTreeData::MergingParams::Aggregating;
const Settings & settings = global_ctx->context->getSettingsRef();
ctx->tasks_for_projections.emplace_back(std::make_shared<MergeTask>(
projection_future_part,
projection.metadata,
global_ctx->merge_entry,
std::make_unique<MergeListElement>((*global_ctx->merge_entry)->table_id, projection_future_part, settings),
std::make_unique<MergeListElement>((*global_ctx->merge_entry)->table_id, projection_future_part, global_ctx->context),
global_ctx->time_of_merge,
global_ctx->context,
global_ctx->space_reservation,

2
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -7420,7 +7420,7 @@ try
part_log_elem.rows = (*merge_entry)->rows_written;
part_log_elem.bytes_uncompressed = (*merge_entry)->bytes_written_uncompressed;
part_log_elem.peak_memory_usage = (*merge_entry)->memory_tracker.getPeak();
part_log_elem.peak_memory_usage = (*merge_entry)->getMemoryTracker().getPeak();
}
if (profile_counters)

13
src/Storages/MergeTree/MutateFromLogEntryTask.cpp
View File

@ -164,21 +164,20 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare()
}
}
const Settings & settings = storage.getContext()->getSettingsRef();
task_context = Context::createCopy(storage.getContext());
task_context->makeQueryContext();
task_context->setCurrentQueryId("");
merge_mutate_entry = storage.getContext()->getMergeList().insert(
storage.getStorageID(),
future_mutated_part,
settings);
task_context);
stopwatch_ptr = std::make_unique<Stopwatch>();
fake_query_context = Context::createCopy(storage.getContext());
fake_query_context->makeQueryContext();
fake_query_context->setCurrentQueryId("");
mutate_task = storage.merger_mutator.mutatePartToTemporaryPart(
future_mutated_part, metadata_snapshot, commands, merge_mutate_entry.get(),
entry.create_time, fake_query_context, NO_TRANSACTION_PTR, reserved_space, table_lock_holder);
entry.create_time, task_context, NO_TRANSACTION_PTR, reserved_space, table_lock_holder);
/// Adjust priority
for (auto & item : future_mutated_part->parts)

1
src/Storages/MergeTree/MutateFromLogEntryTask.h
View File

@ -55,7 +55,6 @@ private:
MergeTreeData::MutableDataPartPtr new_part{nullptr};
FutureMergedMutatedPartPtr future_mutated_part{nullptr};
ContextMutablePtr fake_query_context;
MutateTaskPtr mutate_task;
};

23
src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp
View File

@ -29,11 +29,11 @@ void MutatePlainMergeTreeTask::prepare()
{
future_part = merge_mutate_entry->future_part;
const Settings & settings = storage.getContext()->getSettingsRef();
task_context = createTaskContext();
merge_list_entry = storage.getContext()->getMergeList().insert(
storage.getStorageID(),
future_part,
settings);
task_context);
stopwatch = std::make_unique<Stopwatch>();
@ -52,13 +52,9 @@ void MutatePlainMergeTreeTask::prepare()
std::move(profile_counters_snapshot));
};
fake_query_context = Context::createCopy(storage.getContext());
fake_query_context->makeQueryContext();
fake_query_context->setCurrentQueryId("");
mutate_task = storage.merger_mutator.mutatePartToTemporaryPart(
future_part, metadata_snapshot, merge_mutate_entry->commands, merge_list_entry.get(),
time(nullptr), fake_query_context, merge_mutate_entry->txn, merge_mutate_entry->tagger->reserved_space, table_lock_holder);
time(nullptr), task_context, merge_mutate_entry->txn, merge_mutate_entry->tagger->reserved_space, table_lock_holder);
}
@ -68,9 +64,9 @@ bool MutatePlainMergeTreeTask::executeStep()
ProfileEventsScope profile_events_scope(&profile_counters);
/// Make out memory tracker a parent of current thread memory tracker
MemoryTrackerThreadSwitcherPtr switcher;
std::optional<ThreadGroupSwitcher> switcher;
if (merge_list_entry)
switcher = std::make_unique<MemoryTrackerThreadSwitcher>(*merge_list_entry);
switcher.emplace((*merge_list_entry)->thread_group);
switch (state)
{
@ -130,4 +126,13 @@ bool MutatePlainMergeTreeTask::executeStep()
return false;
}
ContextMutablePtr MutatePlainMergeTreeTask::createTaskContext() const
{
auto context = Context::createCopy(storage.getContext());
context->makeQueryContext();
auto queryId = storage.getStorageID().getShortName() + "::" + future_part->name;
context->setCurrentQueryId(queryId);
return context;
}
}

6
src/Storages/MergeTree/MutatePlainMergeTreeTask.h
View File

@ -74,11 +74,13 @@ private:
std::function<void(const ExecutionStatus & execution_status)> write_part_log;
IExecutableTask::TaskResultCallback task_result_callback;
ContextMutablePtr fake_query_context;
MutateTaskPtr mutate_task;
ProfileEvents::Counters profile_counters;
ContextMutablePtr task_context;
ContextMutablePtr createTaskContext() const;
};

4
src/Storages/MergeTree/MutateTask.cpp
View File

@ -911,14 +911,12 @@ public:
if (projection.type == ProjectionDescription::Type::Aggregate)
projection_merging_params.mode = MergeTreeData::MergingParams::Aggregating;
const Settings & settings = ctx->context->getSettingsRef();
LOG_DEBUG(log, "Merged {} parts in level {} to {}", selected_parts.size(), current_level, projection_future_part->name);
auto tmp_part_merge_task = ctx->mutator->mergePartsToTemporaryPart(
projection_future_part,
projection.metadata,
ctx->mutate_entry,
std::make_unique<MergeListElement>((*ctx->mutate_entry)->table_id, projection_future_part, settings),
std::make_unique<MergeListElement>((*ctx->mutate_entry)->table_id, projection_future_part, ctx->context),
*ctx->holder,
ctx->time_of_mutation,
ctx->context,

4
src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp
View File

@ -128,9 +128,9 @@ bool ReplicatedMergeMutateTaskBase::executeStep()
bool ReplicatedMergeMutateTaskBase::executeImpl()
{
MemoryTrackerThreadSwitcherPtr switcher;
std::optional<ThreadGroupSwitcher> switcher;
if (merge_mutate_entry)
switcher = std::make_unique<MemoryTrackerThreadSwitcher>(*merge_mutate_entry);
switcher.emplace((*merge_mutate_entry)->thread_group);
auto remove_processed_entry = [&] () -> bool
{

1
src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h
View File

@ -62,6 +62,7 @@ protected:
StorageReplicatedMergeTree & storage;
/// ProfileEvents for current part will be stored here
ProfileEvents::Counters profile_counters;
ContextMutablePtr task_context;
private:
enum class CheckExistingPartResult

10
src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
View File

@ -237,7 +237,7 @@ void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPt
context_,
/*can_execute_*/ true,
/*return_all_columns_*/ true,
/*return_deleted_rows_*/ true);
/*return_mutated_rows*/ true);
auto pipeline = QueryPipelineBuilder::getPipeline(interpreter->execute());
PullingPipelineExecutor executor(pipeline);
@ -279,7 +279,13 @@ void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPt
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key cannot be updated (cannot update column {})", primary_key);
auto interpreter = std::make_unique<MutationsInterpreter>(
storage_ptr, metadata_snapshot, commands, context_, /*can_execute_*/ true, /*return_all_columns*/ true);
storage_ptr,
metadata_snapshot,
commands,
context_,
/*can_execute_*/ true,
/*return_all_columns*/ true,
/*return_mutated_rows*/ true);
auto pipeline = QueryPipelineBuilder::getPipeline(interpreter->execute());
PullingPipelineExecutor executor(pipeline);

10
src/Storages/StorageKeeperMap.cpp
View File

@ -864,7 +864,7 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
local_context,
/*can_execute_*/ true,
/*return_all_columns_*/ true,
/*return_deleted_rows_*/ true);
/*return_mutated_rows*/ true);
auto pipeline = QueryPipelineBuilder::getPipeline(interpreter->execute());
PullingPipelineExecutor executor(pipeline);
@ -927,7 +927,13 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key cannot be updated (cannot update column {})", primary_key);
auto interpreter = std::make_unique<MutationsInterpreter>(
storage_ptr, metadata_snapshot, commands, local_context, /*can_execute_*/ true, /*return_all_columns*/ true);
storage_ptr,
metadata_snapshot,
commands,
local_context,
/*can_execute_*/ true,
/*return_all_columns*/ true,
/*return_mutated_rows*/ true);
auto pipeline = QueryPipelineBuilder::getPipeline(interpreter->execute());
PullingPipelineExecutor executor(pipeline);

4
src/Storages/StorageMemory.cpp
View File

@ -406,7 +406,7 @@ namespace
void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional<ASTs> & /* partitions */)
{
auto temp_disk = backup_entries_collector.getContext()->getTemporaryVolume()->getDisk(0);
auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0);
auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size;
backup_entries_collector.addBackupEntries(std::make_shared<MemoryBackup>(
backup_entries_collector.getContext(),
@ -426,7 +426,7 @@ void StorageMemory::restoreDataFromBackup(RestorerFromBackup & restorer, const S
if (!restorer.isNonEmptyTableAllowed() && total_size_bytes)
RestorerFromBackup::throwTableIsNotEmpty(getStorageID());
auto temp_disk = restorer.getContext()->getTemporaryVolume()->getDisk(0);
auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0);
restorer.addDataRestoreTask(
[storage = std::static_pointer_cast<StorageMemory>(shared_from_this()), backup, data_path_in_backup, temp_disk]

2
src/Storages/System/StorageSystemFilesystemCache.cpp
View File

@ -26,6 +26,7 @@ NamesAndTypesList StorageSystemFilesystemCache::getNamesAndTypes()
{"downloaded_size", std::make_shared<DataTypeUInt64>()},
{"persistent", std::make_shared<DataTypeNumber<UInt8>>()},
{"kind", std::make_shared<DataTypeString>()},
{"unbound", std::make_shared<DataTypeNumber<UInt8>>()},
};
}
@ -62,6 +63,7 @@ void StorageSystemFilesystemCache::fillData(MutableColumns & res_columns, Contex
res_columns[8]->insert(file_segment->getDownloadedSize());
res_columns[9]->insert(file_segment->isPersistent());
res_columns[10]->insert(toString(file_segment->getKind()));
res_columns[11]->insert(file_segment->isUnbound());
}
}
}

6
tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference
View File

@ -1,8 +1,8 @@
runtime messages 0.001
runtime exceptions 0.05
messages shorter than 10 0
messages shorter than 16 2
exceptions shorter than 30 27
messages shorter than 10 1
messages shorter than 16 3
exceptions shorter than 30 30
noisy messages 0.3
noisy Trace messages 0.16
noisy Debug messages 0.09

6
tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
View File

@ -53,13 +53,13 @@ create temporary table known_short_messages (s String) as select * from (select
] as arr) array join arr;
-- Check that we don't have too many short meaningless message patterns.
select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_string), 0) from logs where length(message_format_string) < 10 and message_format_string not in known_short_messages;
select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_string), 1) from logs where length(message_format_string) < 10 and message_format_string not in known_short_messages;
-- Same as above. Feel free to update the threshold or remove this query if really necessary
select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 2) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages;
select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages;
-- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary
select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 27) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%' and message_format_string not in known_short_messages;
select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 30) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%' and message_format_string not in known_short_messages;
-- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold

36
tests/queries/0_stateless/00995_order_by_with_fill.reference
View File

@ -1,4 +1,9 @@
*** table without fill to compare ***
--{ echoOn }
DROP TABLE IF EXISTS fill;
CREATE TABLE fill (date Date, val Int, str String) ENGINE = Memory;
INSERT INTO fill VALUES (toDate('2019-05-24'), 13, 'sd0')(toDate('2019-05-10'), 16, 'vp7')(toDate('2019-05-25'), 17, '0ei')(toDate('2019-05-30'), 18, '3kd')(toDate('2019-05-15'), 27, 'enb')(toDate('2019-06-04'), 5, '6az')(toDate('2019-05-23'), 15, '01v')(toDate('2019-05-08'), 28, 'otf')(toDate('2019-05-19'), 20, 'yfh')(toDate('2019-05-07'), 26, '2ke')(toDate('2019-05-07'), 18, 'prh')(toDate('2019-05-09'), 25, '798')(toDate('2019-05-10'), 1, 'myj')(toDate('2019-05-11'), 18, '3s2')(toDate('2019-05-23'), 29, '72y');
-- *** table without fill to compare ***
SELECT * FROM fill ORDER BY date, val;
2019-05-07 18 prh
2019-05-07 26 2ke
2019-05-08 28 otf
@ -14,7 +19,9 @@
2019-05-25 17 0ei
2019-05-30 18 3kd
2019-06-04 5 6az
*** date WITH FILL, val ***
-- Some useful cases
SELECT * FROM fill ORDER BY date WITH FILL, val;
2019-05-07 18 prh
2019-05-07 26 2ke
2019-05-08 28 otf
@ -47,7 +54,7 @@
2019-06-02 0
2019-06-03 0
2019-06-04 5 6az
*** date WITH FILL FROM 2019-05-01 TO 2019-05-31, val WITH FILL ***
SELECT * FROM fill ORDER BY date WITH FILL FROM toDate('2019-05-01') TO toDate('2019-05-31'), val WITH FILL;
2019-05-01 0
2019-05-02 0
2019-05-03 0
@ -116,7 +123,7 @@
2019-05-29 0
2019-05-30 18 3kd
2019-06-04 5 6az
*** date DESC WITH FILL, val WITH FILL FROM 1 TO 6 ***
SELECT * FROM fill ORDER BY date DESC WITH FILL, val WITH FILL FROM 1 TO 6;
2019-06-04 1
2019-06-04 2
2019-06-04 3
@ -275,7 +282,9 @@
2019-05-07 5
2019-05-07 18 prh
2019-05-07 26 2ke
*** date DESC WITH FILL TO 2019-05-01 STEP -2, val DESC WITH FILL FROM 10 TO -5 STEP -3 ***
-- Some weird cases
SELECT * FROM fill ORDER BY date DESC WITH FILL TO toDate('2019-05-01') STEP -2, val DESC WITH FILL FROM 10 TO -5 STEP -3;
2019-06-04 10
2019-06-04 7
2019-06-04 5 6az
@ -376,7 +385,7 @@
2019-05-03 4
2019-05-03 1
2019-05-03 -2
*** date WITH FILL TO 2019-06-23 STEP 3, val WITH FILL FROM -10 STEP 2
SELECT * FROM fill ORDER BY date WITH FILL TO toDate('2019-06-23') STEP 3, val WITH FILL FROM -10 STEP 2;
2019-05-07 -10
2019-05-07 -8
2019-05-07 -6
@ -463,14 +472,18 @@
2019-06-15 -10
2019-06-18 -10
2019-06-21 -10
*** table without fill to compare ***
DROP TABLE fill;
CREATE TABLE fill (a UInt32, b Int32) ENGINE = Memory;
INSERT INTO fill VALUES (1, -2), (1, 3), (3, 2), (5, -1), (6, 5), (8, 0);
-- *** table without fill to compare ***
SELECT * FROM fill ORDER BY a, b;
1 -2
1 3
3 2
5 -1
6 5
8 0
*** a WITH FILL, b WITH fill ***
SELECT * FROM fill ORDER BY a WITH FILL, b WITH fill;
1 -2
1 -1
1 0
@ -484,7 +497,7 @@
6 5
7 0
8 0
*** a WITH FILL, b WITH fill TO 6 STEP 2 ***
SELECT * FROM fill ORDER BY a WITH FILL, b WITH fill TO 6 STEP 2;
1 -2
1 0
1 2
@ -503,3 +516,8 @@
8 0
8 2
8 4
SELECT * FROM fill ORDER BY a WITH FILL STEP -1; -- { serverError 475 }
SELECT * FROM fill ORDER BY a WITH FILL FROM 10 TO 1; -- { serverError 475 }
SELECT * FROM fill ORDER BY a DESC WITH FILL FROM 1 TO 10; -- { serverError 475 }
SELECT * FROM fill ORDER BY a WITH FILL FROM -10 to 10; -- { serverError 475 }
DROP TABLE fill;

12
tests/queries/0_stateless/00995_order_by_with_fill.sql
View File

@ -1,40 +1,34 @@
--{ echoOn }
DROP TABLE IF EXISTS fill;
CREATE TABLE fill (date Date, val Int, str String) ENGINE = Memory;
INSERT INTO fill VALUES (toDate('2019-05-24'), 13, 'sd0')(toDate('2019-05-10'), 16, 'vp7')(toDate('2019-05-25'), 17, '0ei')(toDate('2019-05-30'), 18, '3kd')(toDate('2019-05-15'), 27, 'enb')(toDate('2019-06-04'), 5, '6az')(toDate('2019-05-23'), 15, '01v')(toDate('2019-05-08'), 28, 'otf')(toDate('2019-05-19'), 20, 'yfh')(toDate('2019-05-07'), 26, '2ke')(toDate('2019-05-07'), 18, 'prh')(toDate('2019-05-09'), 25, '798')(toDate('2019-05-10'), 1, 'myj')(toDate('2019-05-11'), 18, '3s2')(toDate('2019-05-23'), 29, '72y');
SELECT '*** table without fill to compare ***';
-- *** table without fill to compare ***
SELECT * FROM fill ORDER BY date, val;
-- Some useful cases
SELECT '*** date WITH FILL, val ***';
SELECT * FROM fill ORDER BY date WITH FILL, val;
SELECT '*** date WITH FILL FROM 2019-05-01 TO 2019-05-31, val WITH FILL ***';
SELECT * FROM fill ORDER BY date WITH FILL FROM toDate('2019-05-01') TO toDate('2019-05-31'), val WITH FILL;
SELECT '*** date DESC WITH FILL, val WITH FILL FROM 1 TO 6 ***';
SELECT * FROM fill ORDER BY date DESC WITH FILL, val WITH FILL FROM 1 TO 6;
-- Some weird cases
SELECT '*** date DESC WITH FILL TO 2019-05-01 STEP -2, val DESC WITH FILL FROM 10 TO -5 STEP -3 ***';
SELECT * FROM fill ORDER BY date DESC WITH FILL TO toDate('2019-05-01') STEP -2, val DESC WITH FILL FROM 10 TO -5 STEP -3;
SELECT '*** date WITH FILL TO 2019-06-23 STEP 3, val WITH FILL FROM -10 STEP 2';
SELECT * FROM fill ORDER BY date WITH FILL TO toDate('2019-06-23') STEP 3, val WITH FILL FROM -10 STEP 2;
DROP TABLE fill;
CREATE TABLE fill (a UInt32, b Int32) ENGINE = Memory;
INSERT INTO fill VALUES (1, -2), (1, 3), (3, 2), (5, -1), (6, 5), (8, 0);
SELECT '*** table without fill to compare ***';
-- *** table without fill to compare ***
SELECT * FROM fill ORDER BY a, b;
SELECT '*** a WITH FILL, b WITH fill ***';
SELECT * FROM fill ORDER BY a WITH FILL, b WITH fill;
SELECT '*** a WITH FILL, b WITH fill TO 6 STEP 2 ***';
SELECT * FROM fill ORDER BY a WITH FILL, b WITH fill TO 6 STEP 2;
SELECT * FROM fill ORDER BY a WITH FILL STEP -1; -- { serverError 475 }

4
tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql
View File

@ -13,9 +13,9 @@ create table data_01641 (key Int, value String) engine=MergeTree order by (key,
SET max_block_size = 1000, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0;
insert into data_01641 select number, toString(number) from numbers(120000);
-- Definitely should fail and it proves that memory is tracked in OPTIMIZE query.
set max_memory_usage='10Mi', max_untracked_memory=0;
optimize table data_01641 final; -- { serverError 241 }
-- It fails iff memory is tracked in OPTIMIZE query, but it doesn't. OPTIMIZE query doesn't rely on query context.
optimize table data_01641 final;
drop table data_01641;

4
tests/queries/0_stateless/01710_projection_with_joins.sql
View File

@ -9,7 +9,7 @@ drop table t;
drop table if exists mt;
create table mt (id1 Int8, id2 Int8) Engine=MergeTree order by tuple();
select id1 as alias1 from mt all inner join (select id2 as alias1 from mt) as t using (alias1) order by id1 settings allow_experimental_projection_optimization = 1;
select alias1 from (select id1, id1 as alias1 from mt) as l all inner join (select id2 as alias1 from mt) as t using (alias1) order by l.id1 settings allow_experimental_projection_optimization = 1;
select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1;
select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) order by id1 settings allow_experimental_projection_optimization = 1;
drop table mt;
@ -17,5 +17,5 @@ drop table mt;
drop table if exists j;
create table j (id1 Int8, id2 Int8, projection p (select id1, id2 order by id2)) Engine=MergeTree order by id1 settings index_granularity = 1;
insert into j select number, number from numbers(10);
select id1 as alias1 from j all inner join (select id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) order by id1 settings allow_experimental_projection_optimization = 1;
select alias1 from (select id1, id1 as alias1 from j) as l all inner join (select id2, id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) order by id1 settings allow_experimental_projection_optimization = 1;
drop table j;

4
tests/queries/0_stateless/01710_projections.sql
View File

@ -7,7 +7,7 @@ insert into projection_test with rowNumberInAllBlocks() as id select 1, toDateTi
set allow_experimental_projection_optimization = 1, force_optimize_projection = 1;
select * from projection_test; -- { serverError 584 }
select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) from projection_test join (select 1) x using (1) where domain = '1' group by dt_m order by dt_m; -- { serverError 584 }
select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) from projection_test join (select 1) x on 1 where domain = '1' group by dt_m order by dt_m; -- { serverError 584 }
select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration) from projection_test where domain = '1' group by dt_m order by dt_m;
@ -39,7 +39,7 @@ select toStartOfMinute(datetime) dt_m, domain, sum(retry_count) / sum(duration),
select toStartOfHour(toStartOfMinute(datetime)) dt_h, uniqHLL12(x_id), uniqHLL12(y_id) from projection_test group by dt_h order by dt_h;
-- found by fuzzer
SET enable_positional_arguments = 0;
SET enable_positional_arguments = 0, force_optimize_projection = 0;
SELECT 2, -1 FROM projection_test PREWHERE domain_alias = 1. WHERE domain = NULL GROUP BY -9223372036854775808 ORDER BY countIf(first_time = 0) / count(-2147483649) DESC NULLS LAST, 1048576 DESC NULLS LAST;
drop table if exists projection_test;

0
tests/queries/0_stateless/01710_projections_and_duplicate_columms.reference Normal file
View File

8
tests/queries/0_stateless/01710_projections_and_duplicate_columms.sql Normal file
View File

@ -0,0 +1,8 @@
drop table if exists projection_test__fuzz_0;
set allow_suspicious_low_cardinality_types=1;
CREATE TABLE projection_test__fuzz_0 (`sum(block_count)` UInt64, `domain_alias` UInt64 ALIAS length(domain), `datetime` DateTime, `domain` LowCardinality(String), `x_id` String, `y_id` String, `block_count` Int64, `retry_count` Int64, `duration` Decimal(76, 13), `kbytes` LowCardinality(Int64), `buffer_time` Int64, `first_time` UInt256, `total_bytes` LowCardinality(Nullable(UInt64)), `valid_bytes` Nullable(UInt64), `completed_bytes` Nullable(UInt64), `fixed_bytes` LowCardinality(Nullable(UInt64)), `force_bytes` Int256, PROJECTION p (SELECT toStartOfMinute(datetime) AS dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) GROUP BY dt_m, domain)) ENGINE = MergeTree PARTITION BY toDate(datetime) ORDER BY (toStartOfTenMinutes(datetime), domain) SETTINGS index_granularity_bytes = 10000000;
INSERT INTO projection_test__fuzz_0 SETTINGS max_threads = 1 WITH rowNumberInAllBlocks() AS id SELECT 1, toDateTime('2020-10-24 00:00:00') + (id / 20), toString(id % 100), * FROM generateRandom('x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64)', 10, 10, 1) LIMIT 1000 SETTINGS max_threads = 1;
SELECT '-21474836.48', 10000000000., '', count(kbytes), '', 10.0001, toStartOfMinute(datetime) AS dt_m, 10, NULL FROM projection_test__fuzz_0 GROUP BY dt_m WITH ROLLUP WITH TOTALS ORDER BY count(retry_count / duration) ASC NULLS LAST, 100000000000000000000. ASC NULLS FIRST format Null;
drop table projection_test__fuzz_0;

3
tests/queries/0_stateless/02458_use_structure_from_insertion_table.reference
View File

@ -1,9 +1,12 @@
\N 0
\N 1
1 2
\N 42
\N 42
\N 42
\N 42
42
42
\N
\N
\N

6
tests/queries/0_stateless/02458_use_structure_from_insertion_table.sql
View File

@ -10,7 +10,7 @@ insert into test select * from file(02458_data.jsonl);
insert into test select x, 1 from file(02458_data.jsonl);
insert into test select x, y from file(02458_data.jsonl);
insert into test select x + 1, y from file(02458_data.jsonl); -- {serverError ONLY_NULLS_WHILE_READING_SCHEMA}
insert into test select x, z from file(02458_data.jsonl); -- {serverError ONLY_NULLS_WHILE_READING_SCHEMA}
insert into test select x, z from file(02458_data.jsonl);
insert into test select * from file(02458_data.jsoncompacteachrow);
insert into test select x, 1 from file(02458_data.jsoncompacteachrow); -- {serverError ONLY_NULLS_WHILE_READING_SCHEMA}
@ -28,8 +28,8 @@ drop table test;
create table test (x Nullable(UInt32)) engine=Memory();
insert into test select * from file(02458_data.jsonl);
insert into test select x from file(02458_data.jsonl);
insert into test select y from file(02458_data.jsonl); -- {serverError ONLY_NULLS_WHILE_READING_SCHEMA}
insert into test select y as x from file(02458_data.jsonl); -- {serverError ONLY_NULLS_WHILE_READING_SCHEMA}
insert into test select y from file(02458_data.jsonl);
insert into test select y as x from file(02458_data.jsonl);
insert into test select c1 from input() format CSV 1,2; -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
insert into test select x from input() format JSONEachRow {"x" : null, "y" : 42}

2
tests/queries/0_stateless/02516_projections_and_context.reference
View File

@ -0,0 +1,2 @@
1
1

4
tests/queries/0_stateless/02516_projections_and_context.sql
View File

@ -1,6 +1,10 @@
DROP TABLE IF EXISTS test1__fuzz_37;
CREATE TABLE test1__fuzz_37 (`i` Date) ENGINE = MergeTree ORDER BY i;
insert into test1__fuzz_37 values ('2020-10-10');
set allow_experimental_analyzer = 0;
SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS }
SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS }
set allow_experimental_analyzer = 1;
SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL));
SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a'));
DROP TABLE test1__fuzz_37;

46
tests/queries/0_stateless/02577_keepermap_delete_update.reference
View File

@ -1,32 +1,32 @@
1 Some string 0
2 Some other string 0
3 random 0
4 random2 0
1 Some string 0 0
2 Some other string 0 0
3 random 0 0
4 random2 0 0
-----------
3 random 0
4 random2 0
3 random 0 0
4 random2 0 0
-----------
3 random 0
3 random 0 0
-----------
0
-----------
1 String 10
2 String 20
3 String 30
4 String 40
1 String 10 0
2 String 20 0
3 String 30 0
4 String 40 0
-----------
1 String 10
2 String 20
3 Another 30
4 Another 40
1 String 10 0
2 String 20 0
3 Another 30 1
4 Another 40 1
-----------
1 String 10
2 String 20
3 Another 30
4 Another 40
1 String 10 0
2 String 20 0
3 Another 30 1
4 Another 40 1
-----------
1 String 102
2 String 202
3 Another 302
4 Another 402
1 String 102 1
2 String 202 1
3 Another 302 2
4 Another 402 2
-----------

40
tests/queries/0_stateless/02577_keepermap_delete_update.sql
View File

@ -1,44 +1,44 @@
-- Tags: no-ordinary-database, no-fasttest
DROP TABLE IF EXISTS 02661_keepermap_delete_update;
DROP TABLE IF EXISTS 02577_keepermap_delete_update;
CREATE TABLE 02661_keepermap_delete_update (key UInt64, value String, value2 UInt64) ENGINE=KeeperMap('/' || currentDatabase() || '/test02661_keepermap_delete_update') PRIMARY KEY(key);
CREATE TABLE 02577_keepermap_delete_update (key UInt64, value String, value2 UInt64) ENGINE=KeeperMap('/' || currentDatabase() || '/test02577_keepermap_delete_update') PRIMARY KEY(key);
INSERT INTO 02661_keepermap_delete_update VALUES (1, 'Some string', 0), (2, 'Some other string', 0), (3, 'random', 0), (4, 'random2', 0);
INSERT INTO 02577_keepermap_delete_update VALUES (1, 'Some string', 0), (2, 'Some other string', 0), (3, 'random', 0), (4, 'random2', 0);
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
DELETE FROM 02661_keepermap_delete_update WHERE value LIKE 'Some%string';
DELETE FROM 02577_keepermap_delete_update WHERE value LIKE 'Some%string';
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02661_keepermap_delete_update DELETE WHERE key >= 4;
ALTER TABLE 02577_keepermap_delete_update DELETE WHERE key >= 4;
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
DELETE FROM 02661_keepermap_delete_update WHERE 1 = 1;
SELECT count() FROM 02661_keepermap_delete_update;
DELETE FROM 02577_keepermap_delete_update WHERE 1 = 1;
SELECT count() FROM 02577_keepermap_delete_update;
SELECT '-----------';
INSERT INTO 02661_keepermap_delete_update VALUES (1, 'String', 10), (2, 'String', 20), (3, 'String', 30), (4, 'String', 40);
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
INSERT INTO 02577_keepermap_delete_update VALUES (1, 'String', 10), (2, 'String', 20), (3, 'String', 30), (4, 'String', 40);
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02661_keepermap_delete_update UPDATE value = 'Another' WHERE key > 2;
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
ALTER TABLE 02577_keepermap_delete_update UPDATE value = 'Another' WHERE key > 2;
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02661_keepermap_delete_update UPDATE key = key * 10 WHERE 1 = 1; -- { serverError BAD_ARGUMENTS }
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
ALTER TABLE 02577_keepermap_delete_update UPDATE key = key * 10 WHERE 1 = 1; -- { serverError BAD_ARGUMENTS }
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02661_keepermap_delete_update UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100;
SELECT * FROM 02661_keepermap_delete_update ORDER BY key;
ALTER TABLE 02577_keepermap_delete_update UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100;
SELECT *, _version FROM 02577_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02661_keepermap_delete_update ON CLUSTER test_shard_localhost UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100; -- { serverError BAD_ARGUMENTS }
ALTER TABLE 02577_keepermap_delete_update ON CLUSTER test_shard_localhost UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100; -- { serverError BAD_ARGUMENTS }
DROP TABLE IF EXISTS 02661_keepermap_delete_update;
DROP TABLE IF EXISTS 02577_keepermap_delete_update;

5
tests/queries/0_stateless/02595_orc_arrow_parquet_more_types.reference Normal file
View File

@ -0,0 +1,5 @@
42 42 42 42 a b
42 42 42 42 a b
42 42 42 42 a b 42.42 0.0.0.0
\N
\N

17
tests/queries/0_stateless/02595_orc_arrow_parquet_more_types.sh Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_LOCAL -q "select 42::Int128 as c1, 42::UInt128 as c2, 42::Int256 as c3, 42::UInt256 as c4, 'a'::Enum8('a' = 1) as c5, 'b'::Enum16('b' = 1) as c6 format Parquet" | $CLICKHOUSE_LOCAL --input-format Parquet --structure="c1 Int128, c2 UInt128, c3 Int256, c4 UInt256, c5 Enum8('a' = 1), c6 Enum16('b' = 1)" -q "select * from table"
$CLICKHOUSE_LOCAL -q "select 42::Int128 as c1, 42::UInt128 as c2, 42::Int256 as c3, 42::UInt256 as c4, 'a'::Enum8('a' = 1) as c5, 'b'::Enum16('b' = 1) as c6 format Arrow" | $CLICKHOUSE_LOCAL --input-format Arrow --structure="c1 Int128, c2 UInt128, c3 Int256, c4 UInt256, c5 Enum8('a' = 1), c6 Enum16('b' = 1)" -q "select * from table"
$CLICKHOUSE_LOCAL -q "select 42::Int128 as c1, 42::UInt128 as c2, 42::Int256 as c3, 42::UInt256 as c4, 'a'::Enum8('a' = 1) as c5, 'b'::Enum16('b' = 1) as c6, 42.42::Decimal256(2) as c7, '0.0.0.0'::IPv4 as c8 format ORC" | $CLICKHOUSE_LOCAL --input-format ORC --structure="c1 Int128, c2 UInt128, c3 Int256, c4 UInt256, c5 Enum8('a' = 1), c6 Enum16('b' = 1), c7 Decimal256(2), c8 IPv4" -q "select * from table"
$CLICKHOUSE_LOCAL -q "select NULL::Nullable(IPv6) as x format ORC" | $CLICKHOUSE_LOCAL --input-format ORC --structure="x Nullable(IPv6)" -q "select * from table"
$CLICKHOUSE_LOCAL -q "select NULL::Nullable(UInt256) as x format ORC" | $CLICKHOUSE_LOCAL --input-format ORC --structure="x Nullable(UInt256)" -q "select * from table"

1
tests/queries/0_stateless/02703_jit_external_aggregation.reference Normal file
View File

@ -0,0 +1 @@
.....

35
tests/queries/0_stateless/02703_jit_external_aggregation.sh Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Tags: long, no-asan, no-msan, no-tsan, no-ubsan
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
# This query should return empty result in every of five runs:
for _ in {1..5}
do
$CLICKHOUSE_CLIENT --compile_aggregate_expressions 0 --query "
SELECT
COUNT() AS c,
group_key,
anyIf(r, key = 0) AS x0,
anyIf(r, key = 1) AS x1,
anyIf(r, key = 2) AS x2
FROM
(
SELECT
CRC32(toString(number)) % 1000000 AS group_key,
number % 3 AS key,
number AS r
FROM numbers(10000000)
)
GROUP BY group_key
HAVING (c = 2) AND (x0 > 0) AND (x1 > 0) AND (x2 > 0)
ORDER BY group_key ASC
LIMIT 10
SETTINGS max_bytes_before_external_group_by = 200000
" && echo -n '.'
done
echo

46
tests/queries/0_stateless/02707_keeper_map_delete_update_strict.reference
View File

@ -1,32 +1,32 @@
1 Some string 0
2 Some other string 0
3 random 0
4 random2 0
1 Some string 0 0 0
2 Some other string 0 0 0
3 random 0 0 0
4 random2 0 0 0
-----------
3 random 0
4 random2 0
3 random 0 0
4 random2 0 0
-----------
3 random 0
3 random 0 0
-----------
0
-----------
1 String 10
2 String 20
3 String 30
4 String 40
1 String 10 0
2 String 20 0
3 String 30 0
4 String 40 0
-----------
1 String 10
2 String 20
3 Another 30
4 Another 40
1 String 10 0
2 String 20 0
3 Another 30 1
4 Another 40 1
-----------
1 String 10
2 String 20
3 Another 30
4 Another 40
1 String 10 0
2 String 20 0
3 Another 30 1
4 Another 40 1
-----------
1 String 102
2 String 202
3 Another 302
4 Another 402
1 String 102 1
2 String 202 1
3 Another 302 2
4 Another 402 2
-----------

14
tests/queries/0_stateless/02707_keeper_map_delete_update_strict.sql
View File

@ -8,17 +8,17 @@ CREATE TABLE 02707_keepermap_delete_update (key UInt64, value String, value2 UIn
INSERT INTO 02707_keepermap_delete_update VALUES (1, 'Some string', 0), (2, 'Some other string', 0), (3, 'random', 0), (4, 'random2', 0);
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
DELETE FROM 02707_keepermap_delete_update WHERE value LIKE 'Some%string';
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02707_keepermap_delete_update DELETE WHERE key >= 4;
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
DELETE FROM 02707_keepermap_delete_update WHERE 1 = 1;
@ -26,19 +26,19 @@ SELECT count() FROM 02707_keepermap_delete_update;
SELECT '-----------';
INSERT INTO 02707_keepermap_delete_update VALUES (1, 'String', 10), (2, 'String', 20), (3, 'String', 30), (4, 'String', 40);
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02707_keepermap_delete_update UPDATE value = 'Another' WHERE key > 2;
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02707_keepermap_delete_update UPDATE key = key * 10 WHERE 1 = 1; -- { serverError 36 }
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
ALTER TABLE 02707_keepermap_delete_update UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100;
SELECT * FROM 02707_keepermap_delete_update ORDER BY key;
SELECT *, _version FROM 02707_keepermap_delete_update ORDER BY key;
SELECT '-----------';
DROP TABLE IF EXISTS 02707_keepermap_delete_update;