thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into update_cassandra

Browse Source
This commit is contained in:
Alexander Tokmakov 2021-01-20 14:15:53 +03:00
commit 0507a38ec6
57 changed files with 1295 additions and 646 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -12,6 +12,8 @@
* Removed aggregate functions `timeSeriesGroupSum`, `timeSeriesGroupRateSum` because a friend of mine said they never worked. This fixes [#16869](https://github.com/ClickHouse/ClickHouse/issues/16869). If you have luck using these functions, write a email to clickhouse-feedback@yandex-team.com. [#17423](https://github.com/ClickHouse/ClickHouse/pull/17423) ([alexey-milovidov](https://github.com/alexey-milovidov)).
* Prohibit toUnixTimestamp(Date()) (before it just returns UInt16 representation of Date). [#17376](https://github.com/ClickHouse/ClickHouse/pull/17376) ([Azat Khuzhin](https://github.com/azat)).
* Allow using extended integer types (`Int128`, `Int256`, `UInt256`) in `avg` and `avgWeighted` functions. Also allow using different types (integer, decimal, floating point) for value and for weight in `avgWeighted` function. This is a backward-incompatible change: now the `avg` and `avgWeighted` functions always return `Float64` (as documented). Before this change the return type for `Decimal` arguments was also `Decimal`. [#15419](https://github.com/ClickHouse/ClickHouse/pull/15419) ([Mike](https://github.com/myrrc)).
* Expression `toUUID(N)` no longer works. Replace with `toUUID('00000000-0000-0000-0000-000000000000')`. This change is motivated by non-obvious results of `toUUID(N)` where N is non zero.
* SSL Certificates with incorrect "key usage" are rejected. In previous versions they are used to work. See [#19262](https://github.com/ClickHouse/ClickHouse/issues/19262).
#### New Feature

2
docker/test/fasttest/run.sh
View File

@ -335,7 +335,7 @@ function run_tests
time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
# substr is to remove semicolon after test name
readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt")
readarray -t FAILED_TESTS < <(awk '/\[ FAIL|TIMEOUT|ERROR \]/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt")
# We will rerun sequentially any tests that have failed during parallel run.
# They might have failed because there was some interference from other tests

4
docker/test/fuzzer/run-fuzzer.sh
View File

@ -75,7 +75,7 @@ function fuzz
{
# Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
cd ch
NEW_TESTS=$(git diff --name-only master | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R)
NEW_TESTS=$(git diff --name-only master "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R)
cd ..
if [[ -n "$NEW_TESTS" ]]
then
@ -175,7 +175,7 @@ case "$stage" in
# Lost connection to the server. This probably means that the server died
# with abort.
echo "failure" > status.txt
if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*" server.log > description.txt
if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*" server.log > description.txt
then
echo "Lost connection to server. See the logs" > description.txt
fi

21
docker/test/sqlancer/run.sh
View File

@ -10,6 +10,23 @@ dpkg -i package_folder/clickhouse-client_*.deb
service clickhouse-server start && sleep 5
cd /sqlancer/sqlancer-master
CLICKHOUSE_AVAILABLE=true mvn -Dtest=TestClickHouse test
cp /sqlancer/sqlancer-master/target/surefire-reports/TEST-sqlancer.dbms.TestClickHouse.xml /test_output/result.xml
export TIMEOUT=60
export NUM_QUERIES=1000
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPWhere | tee /test_output/TLPWhere.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPWhere.err
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPGroupBy | tee /test_output/TLPGroupBy.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPGroupBy.err
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPHaving | tee /test_output/TLPHaving.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPHaving.err
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPWhere --oracle TLPGroupBy | tee /test_output/TLPWhereGroupBy.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPWhereGroupBy.err
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPDistinct | tee /test_output/TLPDistinct.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPDistinct.err
( java -jar target/SQLancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPAggregate | tee /test_output/TLPAggregate.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPAggregate.err
service clickhouse-server stop && sleep 10
ls /var/log/clickhouse-server/
tar czf /test_output/logs.tar.gz -C /var/log/clickhouse-server/ .
tail -n 1000 /var/log/clickhouse-server/stderr.log > /test_output/stderr.log
tail -n 1000 /var/log/clickhouse-server/stdout.log > /test_output/stdout.log
tail -n 1000 /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log
ls /test_output

4
docs/en/interfaces/third-party/gui.md vendored
View File

@ -107,6 +107,10 @@ Features:
[xeus-clickhouse](https://github.com/wangfenjin/xeus-clickhouse) is a Jupyter kernal for ClickHouse, which supports query CH data using SQL in Jupyter.
### MindsDB Studio {#mindsdb}
[MindsDB](https://mindsdb.com/) is an open-source AI layer for databases including ClickHouse that allows you to effortlessly develop, train and deploy state-of-the-art machine learning models. MindsDB Studio(GUI) allows you to train new models from database, interpret predictions made by the model, identify potential data biases, and evaluate and visualize model accuracy using the Explainable AI function to adapt and tune your Machine Learning models faster.
## Commercial {#commercial}
### DataGrip {#datagrip}

3
docs/en/interfaces/third-party/integrations.md vendored
View File

@ -69,6 +69,9 @@ toc_title: Integrations
- Geo
- [MaxMind](https://dev.maxmind.com/geoip/)
- [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
- AutoML
- [MindsDB](https://mindsdb.com/)
- [MindsDB](https://github.com/mindsdb/mindsdb) - Predictive AI layer for ClickHouse database.
## Programming Language Ecosystems {#programming-language-ecosystems}

15
docs/en/operations/settings/settings.md
View File

@ -2134,6 +2134,21 @@ Default value: `1`.
- [ORDER BY Clause](../../sql-reference/statements/select/order-by.md#optimize_read_in_order)
## optimize_aggregation_in_order {#optimize_aggregation_in_order}
Enables [GROUP BY](../../sql-reference/statements/select/group-by.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries for aggregating data in corresponding order in [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables.
Possible values:
- 0 — `GROUP BY` optimization is disabled.
- 1 — `GROUP BY` optimization is enabled.
Default value: `0`.
**See Also**
- [GROUP BY optimization](../../sql-reference/statements/select/group-by.md#aggregation-in-order)
## mutations_sync {#mutations_sync}
Allows to execute `ALTER TABLE ... UPDATE|DELETE` queries ([mutations](../../sql-reference/statements/alter/index.md#mutations)) synchronously.

54
docs/en/sql-reference/ansi.md
View File

@ -25,22 +25,22 @@ The following table lists cases when query feature works in ClickHouse, but beha
|------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **E011** | **Numeric data types** | **Partial**{.text-warning} | |
| E011-01 | INTEGER and SMALLINT data types | Yes {.text-success} | |
| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Partial {.text-warning} | `FLOAT(<binary_precision>)`, `REAL` and `DOUBLE PRECISION` are not supported |
| E011-03 | DECIMAL and NUMERIC data types | Partial {.text-warning} | Only `DECIMAL(p,s)` is supported, not `NUMERIC` |
| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Yes {.text-success} | |
| E011-03 | DECIMAL and NUMERIC data types | Yes {.text-success} | |
| E011-04 | Arithmetic operators | Yes {.text-success} | |
| E011-05 | Numeric comparison | Yes {.text-success} | |
| E011-06 | Implicit casting among the numeric data types | No {.text-danger} | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
| **E021** | **Character string types** | **Partial**{.text-warning} | |
| E021-01 | CHARACTER data type | No {.text-danger} | |
| E021-02 | CHARACTER VARYING data type | No {.text-danger} | `String` behaves similarly, but without length limit in parentheses |
| E021-03 | Character literals | Partial {.text-warning} | No automatic concatenation of consecutive literals and character set support |
| E021-01 | CHARACTER data type | Yes {.text-success} | |
| E021-02 | CHARACTER VARYING data type | Yes {.text-success} | |
| E021-03 | Character literals | Yes {.text-success} | |
| E021-04 | CHARACTER_LENGTH function | Partial {.text-warning} | No `USING` clause |
| E021-05 | OCTET_LENGTH function | No {.text-danger} | `LENGTH` behaves similarly |
| E021-06 | SUBSTRING | Partial {.text-warning} | No support for `SIMILAR` and `ESCAPE` clauses, no `SUBSTRING_REGEX` variant |
| E021-07 | Character concatenation | Partial {.text-warning} | No `COLLATE` clause |
| E021-08 | UPPER and LOWER functions | Yes {.text-success} | |
| E021-09 | TRIM function | Yes {.text-success} | |
| E021-10 | Implicit casting among the fixed-length and variable-length character string types | No {.text-danger} | ANSI SQL allows arbitrary implicit cast between string types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
| E021-10 | Implicit casting among the fixed-length and variable-length character string types | Partial | ANSI SQL allows arbitrary implicit cast between string types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
| E021-11 | POSITION function | Partial {.text-warning} | No support for `IN` and `USING` clauses, no `POSITION_REGEX` variant |
| E021-12 | Character comparison | Yes {.text-success} | |
| **E031** | **Identifiers** | **Partial**{.text-warning} | |
@ -71,20 +71,20 @@ The following table lists cases when query feature works in ClickHouse, but beha
| E061-13 | Correlated subqueries | No {.text-danger} | |
| E061-14 | Search condition | Yes {.text-success} | |
| **E071** | **Basic query expressions** | **Partial**{.text-warning} | |
| E071-01 | UNION DISTINCT table operator | No {.text-danger} | |
| E071-01 | UNION DISTINCT table operator | Yes {.text-success} | |
| E071-02 | UNION ALL table operator | Yes {.text-success} | |
| E071-03 | EXCEPT DISTINCT table operator | No {.text-danger} | |
| E071-05 | Columns combined via table operators need not have exactly the same data type | Yes {.text-success} | |
| E071-06 | Table operators in subqueries | Yes {.text-success} | |
| **E081** | **Basic privileges** | **Partial**{.text-warning} | Work in progress |
| E081-01 | SELECT privilege at the table level | | |
| **E081** | **Basic privileges** | **Yes**{.text-success} | |
| E081-01 | SELECT privilege at the table level | Yes {.text-success} | |
| E081-02 | DELETE privilege | | |
| E081-03 | INSERT privilege at the table level | | |
| E081-04 | UPDATE privilege at the table level | | |
| E081-03 | INSERT privilege at the table level | Yes {.text-success} | |
| E081-04 | UPDATE privilege at the table level | Yes {.text-success} | |
| E081-05 | UPDATE privilege at the column level | | |
| E081-06 | REFERENCES privilege at the table level | | |
| E081-07 | REFERENCES privilege at the column level | | |
| E081-08 | WITH GRANT OPTION | | |
| E081-08 | WITH GRANT OPTION | Yes {.text-success} | |
| E081-09 | USAGE privilege | | |
| E081-10 | EXECUTE privilege | | |
| **E091** | **Set functions** | **Yes**{.text-success} | |
@ -93,28 +93,28 @@ The following table lists cases when query feature works in ClickHouse, but beha
| E091-03 | MAX | Yes {.text-success} | |
| E091-04 | MIN | Yes {.text-success} | |
| E091-05 | SUM | Yes {.text-success} | |
| E091-06 | ALL quantifier | No {.text-danger} | |
| E091-07 | DISTINCT quantifier | Partial {.text-warning} | Not all aggregate functions supported |
| E091-06 | ALL quantifier | Yes {.text-success} | |
| E091-07 | DISTINCT quantifier | Yes {.text-success} | Not all aggregate functions supported |
| **E101** | **Basic data manipulation** | **Partial**{.text-warning} | |
| E101-01 | INSERT statement | Yes {.text-success} | Note: primary key in ClickHouse does not imply the `UNIQUE` constraint |
| E101-03 | Searched UPDATE statement | No {.text-danger} | There’s an `ALTER UPDATE` statement for batch data modification |
| E101-04 | Searched DELETE statement | No {.text-danger} | There’s an `ALTER DELETE` statement for batch data removal |
| E101-03 | Searched UPDATE statement | Partial | There’s an `ALTER UPDATE` statement for batch data modification |
| E101-04 | Searched DELETE statement | Partial | There’s an `ALTER DELETE` statement for batch data removal |
| **E111** | **Single row SELECT statement** | **No**{.text-danger} | |
| **E121** | **Basic cursor support** | **No**{.text-danger} | |
| E121-01 | DECLARE CURSOR | No {.text-danger} | |
| E121-02 | ORDER BY columns need not be in select list | No {.text-danger} | |
| E121-03 | Value expressions in ORDER BY clause | No {.text-danger} | |
| E121-02 | ORDER BY columns need not be in select list | Yes {.text-success} | |
| E121-03 | Value expressions in ORDER BY clause | Yes {.text-success} | |
| E121-04 | OPEN statement | No {.text-danger} | |
| E121-06 | Positioned UPDATE statement | No {.text-danger} | |
| E121-07 | Positioned DELETE statement | No {.text-danger} | |
| E121-08 | CLOSE statement | No {.text-danger} | |
| E121-10 | FETCH statement: implicit NEXT | No {.text-danger} | |
| E121-17 | WITH HOLD cursors | No {.text-danger} | |
| **E131** | **Null value support (nulls in lieu of values)** | **Partial**{.text-warning} | Some restrictions apply |
| **E131** | **Null value support (nulls in lieu of values)** | **Yes**{.text-success} | Some restrictions apply |
| **E141** | **Basic integrity constraints** | **Partial**{.text-warning} | |
| E141-01 | NOT NULL constraints | Yes {.text-success} | Note: `NOT NULL` is implied for table columns by default |
| E141-02 | UNIQUE constraint of NOT NULL columns | No {.text-danger} | |
| E141-03 | PRIMARY KEY constraints | No {.text-danger} | |
| E141-03 | PRIMARY KEY constraints | Partial | |
| E141-04 | Basic FOREIGN KEY constraint with the NO ACTION default for both referential delete action and referential update action | No {.text-danger} | |
| E141-06 | CHECK constraint | Yes {.text-success} | |
| E141-07 | Column defaults | Yes {.text-success} | |
@ -126,7 +126,7 @@ The following table lists cases when query feature works in ClickHouse, but beha
| **E152** | **Basic SET TRANSACTION statement** | **No**{.text-danger} | |
| E152-01 | SET TRANSACTION statement: ISOLATION LEVEL SERIALIZABLE clause | No {.text-danger} | |
| E152-02 | SET TRANSACTION statement: READ ONLY and READ WRITE clauses | No {.text-danger} | |
| **E153** | **Updatable queries with subqueries** | **No**{.text-danger} | |
| **E153** | **Updatable queries with subqueries** | **Yes**{.text-success} | |
| **E161** | **SQL comments using leading double minus** | **Yes**{.text-success} | |
| **E171** | **SQLSTATE support** | **No**{.text-danger} | |
| **E182** | **Host language binding** | **No**{.text-danger} | |
@ -134,7 +134,7 @@ The following table lists cases when query feature works in ClickHouse, but beha
| F031-01 | CREATE TABLE statement to create persistent base tables | Partial {.text-warning} | No `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` clauses and no support for user resolved data types |
| F031-02 | CREATE VIEW statement | Partial {.text-warning} | No `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` clauses and no support for user resolved data types |
| F031-03 | GRANT statement | Yes {.text-success} | |
| F031-04 | ALTER TABLE statement: ADD COLUMN clause | Partial {.text-warning} | No support for `GENERATED` clause and system time period |
| F031-04 | ALTER TABLE statement: ADD COLUMN clause | Yes {.text-success} | No support for `GENERATED` clause and system time period |
| F031-13 | DROP TABLE statement: RESTRICT clause | No {.text-danger} | |
| F031-16 | DROP VIEW statement: RESTRICT clause | No {.text-danger} | |
| F031-19 | REVOKE statement: RESTRICT clause | No {.text-danger} | |
@ -147,11 +147,11 @@ The following table lists cases when query feature works in ClickHouse, but beha
| F041-07 | The inner table in a left or right outer join can also be used in an inner join | Yes {.text-success} | |
| F041-08 | All comparison operators are supported (rather than just =) | No {.text-danger} | |
| **F051** | **Basic date and time** | **Partial**{.text-warning} | |
| F051-01 | DATE data type (including support of DATE literal) | Partial {.text-warning} | No literal |
| F051-01 | DATE data type (including support of DATE literal) | Yes {.text-success} | |
| F051-02 | TIME data type (including support of TIME literal) with fractional seconds precision of at least 0 | No {.text-danger} | |
| F051-03 | TIMESTAMP data type (including support of TIMESTAMP literal) with fractional seconds precision of at least 0 and 6 | No {.text-danger} | `DateTime64` time provides similar functionality |
| F051-04 | Comparison predicate on DATE, TIME, and TIMESTAMP data types | Partial {.text-warning} | Only one data type available |
| F051-05 | Explicit CAST between datetime types and character string types | Yes {.text-success} | |
| F051-03 | TIMESTAMP data type (including support of TIMESTAMP literal) with fractional seconds precision of at least 0 and 6 | Yes {.text-success} | |
| F051-04 | Comparison predicate on DATE, TIME, and TIMESTAMP data types | Yes {.text-success} | |
| F051-05 | Explicit CAST between datetime types and character string types | Yes {.text-success} | |
| F051-06 | CURRENT_DATE | No {.text-danger} | `today()` is similar |
| F051-07 | LOCALTIME | No {.text-danger} | `now()` is similar |
| F051-08 | LOCALTIMESTAMP | No {.text-danger} | |
@ -171,7 +171,7 @@ The following table lists cases when query feature works in ClickHouse, but beha
| F261-03 | NULLIF | Yes {.text-success} | |
| F261-04 | COALESCE | Yes {.text-success} | |
| **F311** | **Schema definition statement** | **Partial**{.text-warning} | |
| F311-01 | CREATE SCHEMA | No {.text-danger} | |
| F311-01 | CREATE SCHEMA | Partial {.text-danger} | See CREATE DATABASE |
| F311-02 | CREATE TABLE for persistent base tables | Yes {.text-success} | |
| F311-03 | CREATE VIEW | Yes {.text-success} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | No {.text-danger} | |

4
docs/en/sql-reference/statements/select/group-by.md
View File

@ -255,6 +255,10 @@ For every different key value encountered, `GROUP BY` calculates a set of aggreg
Aggregation is one of the most important features of a column-oriented DBMS, and thus it’s implementation is one of the most heavily optimized parts of ClickHouse. By default, aggregation is done in memory using a hash-table. It has 40+ specializations that are chosen automatically depending on “grouping key” data types.
### GROUP BY Optimization Depending on Table Sorting Key {#aggregation-in-order}
The aggregation can be performed more effectively, if a table is sorted by some key, and `GROUP BY` expression contains at least prefix of sorting key or injective functions. In this case when a new key is read from table, the in-between result of aggregation can be finalized and sent to client. This behaviour is switched on by the [optimize_aggregation_in_order](../../../operations/settings/settings.md#optimize_aggregation_in_order) setting. Such optimization reduces memory usage during aggregation, but in some cases may slow down the query execution.
### GROUP BY in External Memory {#select-group-by-in-external-memory}
You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`.

5
docs/en/whats-new/changelog/2020.md
View File

@ -1,3 +1,8 @@
---
toc_priority: 76
toc_title: '2020'
---
### ClickHouse release 20.12
### ClickHouse release v20.12.4.5-stable, 2020-12-24

15
docs/ru/operations/settings/settings.md
View File

@ -1995,6 +1995,21 @@ SELECT * FROM a;
- [Оптимизация чтения данных](../../sql-reference/statements/select/order-by.md#optimize_read_in_order) в секции `ORDER BY`
## optimize_aggregation_in_order {#optimize_aggregation_in_order}
Включает или отключает оптимизацию в запросах [SELECT](../../sql-reference/statements/select/index.md) с секцией [GROUP BY](../../sql-reference/statements/select/group-by.md) при наличии подходящих ключей сортировки. Используется при работе с таблицами [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md).
Возможные значения:
- 0 — оптимизация по ключу сортировки отключена.
- 1 — оптимизация по ключу сортировки включена.
Значение по умолчанию: `0`.
**См. также**
- [Оптимизация GROUP BY для отсортированных таблиц](../../sql-reference/statements/select/group-by.md#aggregation-in-order)
## mutations_sync {#mutations_sync}
Позволяет выполнять запросы `ALTER TABLE ... UPDATE|DELETE` ([мутации](../../sql-reference/statements/alter/index.md#mutations)) синхронно.

4
docs/ru/sql-reference/statements/select/group-by.md
View File

@ -252,6 +252,10 @@ GROUP BY вычисляет для каждого встретившегося
Агрегация является одной из наиболее важных возможностей столбцовых СУБД, и поэтому её реализация является одной из наиболее сильно оптимизированных частей ClickHouse. По умолчанию агрегирование выполняется в памяти с помощью хэш-таблицы. Она имеет более 40 специализаций, которые выбираются автоматически в зависимости от типов данных ключа группировки.
### Оптимизация GROUP BY для отсортированных таблиц {#aggregation-in-order}
Агрегирование данных в отсортированных таблицах может выполняться более эффективно, если выражение `GROUP BY` содержит хотя бы префикс ключа сортировки или инъективную функцию с этим ключом. В таких случаях в момент считывания из таблицы нового значения ключа сортировки промежуточный результат агрегирования будет финализироваться и отправляться на клиентскую машину. Чтобы включить такой способ выполнения запроса, используйте настройку [optimize_aggregation_in_order](../../../operations/settings/settings.md#optimize_aggregation_in_order). Подобная оптимизация позволяет сэкономить память во время агрегации, но в некоторых случаях может привести к увеличению времени выполнения запроса.
### Группировка во внешней памяти {#select-group-by-in-external-memory}
Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`.

130
docs/zh/sql-reference/ansi.md
View File

@ -12,18 +12,18 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
## 行为差异 {#differences-in-behaviour}
下表列出了查询功能在ClickHouse中工作但行为不符合ANSI SQL中指定的情况。
下表列出了查询功能在ClickHouse中有效但不符合ANSI SQL标准的情况。
| Feature ID | 功能名称 | 差异 |
|------------|--------------------|---------------------------------------------------------------------|
| E011 | 数字数据类型 | 带句点的数值文字被解释为近似值 (`Float64`）而不是确切的 (`Decimal`) |
| E051-05 | 选择项目可以重命名 | 项目重命名具有比仅选择结果更广泛的可见性范围 |
| E141-01 | 非空约束 | `NOT NULL` 默认情况下，表列隐含 |
| E011-04 | 算术运算符 | ClickHouse溢出而不是检查算法，并根据自定义规则更改结果数据类型 |
| E011 | 数值（Numeric）数据类型 | 带小数点的数值文字被解释为近似值 (`Float64`）而不是精确值 (`Decimal`) |
| E051-05 | SELECT字段可以重命名 | 字段不仅仅在SELECT结果中可被重命名 |
| E141-01 | 非空约束 | 表中每一列默认为`NOT NULL` |
| E011-04 | 算术运算符 | ClickHouse不会检查算法，并根据自定义规则更改结果数据类型，而是会溢出 |
## 功能状态 {#feature-status}
## 功能匹配 {#feature-status}
| Feature ID | 功能名称 | 状态 | 评论 |
| Feature ID | 功能名称 | 匹配 | 评论 |
|------------|----------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **E011** | **数字数据类型** | **部分**{.text-warning} | |
| E011-01 | 整型和小型数据类型 | 是 {.text-success} | |
@ -31,10 +31,10 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| E011-03 | 十进制和数值数据类型 | 部分 {.text-warning} | 只有 `DECIMAL(p,s)` 支持，而不是 `NUMERIC` |
| E011-04 | 算术运算符 | 是 {.text-success} | |
| E011-05 | 数字比较 | 是 {.text-success} | |
| E011-06 | 数字数据类型之间的隐式转换 | 非也。 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| E011-06 | 数字数据类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| **E021** | **字符串类型** | **部分**{.text-warning} | |
| E021-01 | 字符数据类型 | 非也。 {.text-danger} | |
| E021-02 | 字符变化数据类型 | 非也。 {.text-danger} | `String` 行为类似，但括号中没有长度限制 |
| E021-01 | 字符数据类型 | 否。 {.text-danger} | |
| E021-02 | 字符变化数据类型 | 否。 {.text-danger} | `String` 行为类似，但括号中没有长度限制 |
| E021-03 | 字符文字 | 部分 {.text-warning} | 不自动连接连续文字和字符集支持 |
| E021-04 | 字符长度函数 | 部分 {.text-warning} | 非也。 `USING` 条款 |
| E021-05 | OCTET_LENGTH函数 | 非也。 {.text-danger} | `LENGTH` 表现类似 |
@ -42,7 +42,7 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| E021-07 | 字符串联 | 部分 {.text-warning} | 非也。 `COLLATE` 条款 |
| E021-08 | 上下功能 | 是 {.text-success} | |
| E021-09 | 修剪功能 | 是 {.text-success} | |
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 非也。 {.text-danger} | ANSI SQL允许在字符串类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在字符串类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| E021-11 | 职位功能 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 条款，否 `POSITION_REGEX` 备选案文 |
| E021-12 | 字符比较 | 是 {.text-success} | |
| **E031** | **标识符** | **部分**{.text-warning} | |
@ -57,23 +57,23 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| E051-06 | 有条款 | 是 {.text-success} | |
| E051-07 | 合格\*在选择列表中 | 是 {.text-success} | |
| E051-08 | FROM子句中的关联名称 | 是 {.text-success} | |
| E051-09 | 重命名FROM子句中的列 | 非也。 {.text-danger} | |
| E051-09 | 重命名FROM子句中的列 | 否。 {.text-danger} | |
| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | |
| E061-01 | 比较谓词 | 是 {.text-success} | |
| E061-02 | 谓词之间 | 部分 {.text-warning} | 非也。 `SYMMETRIC` 和 `ASYMMETRIC` 条款 |
| E061-03 | 在具有值列表的谓词中 | 是 {.text-success} | |
| E061-04 | 像谓词 | 是 {.text-success} | |
| E061-05 | LIKE谓词：逃避条款 | 非也。 {.text-danger} | |
| E061-05 | LIKE谓词：逃避条款 | 否。 {.text-danger} | |
| E061-06 | 空谓词 | 是 {.text-success} | |
| E061-07 | 量化比较谓词 | 非也。 {.text-danger} | |
| E061-08 | 存在谓词 | 非也。 {.text-danger} | |
| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | |
| E061-11 | 谓词中的子查询 | 是 {.text-success} | |
| E061-12 | 量化比较谓词中的子查询 | 非也。 {.text-danger} | |
| E061-13 | 相关子查询 | 非也。 {.text-danger} | |
| E061-12 | 量化比较谓词中的子查询 | 否。 {.text-danger} | |
| E061-13 | 相关子查询 | 否。 {.text-danger} | |
| E061-14 | 搜索条件 | 是 {.text-success} | |
| **E071** | **基本查询表达式** | **部分**{.text-warning} | |
| E071-01 | UNION DISTINCT table运算符 | 非也。 {.text-danger} | |
| E071-01 | UNION DISTINCT table运算符 | 否。 {.text-danger} | |
| E071-02 | 联合所有表运算符 | 是 {.text-success} | |
| E071-03 | 除了不同的表运算符 | 非也。 {.text-danger} | |
| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | |
@ -85,51 +85,51 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| E091-03 | MAX | 是 {.text-success} | |
| E091-04 | MIN | 是 {.text-success} | |
| E091-05 | SUM | 是 {.text-success} | |
| E091-06 | 全部量词 | 非也。 {.text-danger} | |
| E091-06 | 全部量词 | 否。 {.text-danger} | |
| E091-07 | 不同的量词 | 部分 {.text-warning} | 并非所有聚合函数都受支持 |
| **E101** | **基本数据操作** | **部分**{.text-warning} | |
| E101-01 | 插入语句 | 是 {.text-success} | 注：ClickHouse中的主键并不意味着 `UNIQUE` 约束 |
| E101-03 | 搜索更新语句 | 非也。 {.text-danger} | 有一个 `ALTER UPDATE` 批量数据修改语句 |
| E101-04 | 搜索的删除语句 | 非也。 {.text-danger} | 有一个 `ALTER DELETE` 批量数据删除声明 |
| **E111** | **单行SELECT语句** | **非也。**{.text-danger} | |
| **E121** | **基本光标支持** | **非也。**{.text-danger} | |
| E121-01 | DECLARE CURSOR | 非也。 {.text-danger} | |
| E121-02 | 按列排序不需要在选择列表中 | 非也。 {.text-danger} | |
| E121-03 | 按顺序排列的值表达式 | 非也。 {.text-danger} | |
| E121-04 | 公开声明 | 非也。 {.text-danger} | |
| E121-06 | 定位更新语句 | 非也。 {.text-danger} | |
| E121-07 | 定位删除语句 | 非也。 {.text-danger} | |
| E121-08 | 关闭声明 | 非也。 {.text-danger} | |
| E121-10 | FETCH语句：隐式NEXT | 非也。 {.text-danger} | |
| E121-17 | 使用保持游标 | 非也。 {.text-danger} | |
| E101-03 | 搜索更新语句 | 否。 {.text-danger} | 有一个 `ALTER UPDATE` 批量数据修改语句 |
| E101-04 | 搜索的删除语句 | 否。 {.text-danger} | 有一个 `ALTER DELETE` 批量数据删除声明 |
| **E111** | **单行SELECT语句** | **否。**{.text-danger} | |
| **E121** | **基本光标支持** | **否。**{.text-danger} | |
| E121-01 | DECLARE CURSOR | 否。 {.text-danger} | |
| E121-02 | 按列排序不需要在选择列表中 | 否。 {.text-danger} | |
| E121-03 | 按顺序排列的值表达式 | 否。 {.text-danger} | |
| E121-04 | 公开声明 | 否。 {.text-danger} | |
| E121-06 | 定位更新语句 | 否。 {.text-danger} | |
| E121-07 | 定位删除语句 | 否。 {.text-danger} | |
| E121-08 | 关闭声明 | 否。 {.text-danger} | |
| E121-10 | FETCH语句：隐式NEXT | 否。 {.text-danger} | |
| E121-17 | 使用保持游标 | 否。 {.text-danger} | |
| **E131** | **空值支持（空值代替值)** | **部分**{.text-warning} | 一些限制适用 |
| **E141** | **基本完整性约束** | **部分**{.text-warning} | |
| E141-01 | 非空约束 | 是 {.text-success} | 注: `NOT NULL` 默认情况下，表列隐含 |
| E141-02 | 非空列的唯一约束 | 非也。 {.text-danger} | |
| E141-03 | 主键约束 | 非也。 {.text-danger} | |
| E141-04 | 对于引用删除操作和引用更新操作，具有默认无操作的基本外键约束 | 非也。 {.text-danger} | |
| E141-02 | 非空列的唯一约束 | 否。 {.text-danger} | |
| E141-03 | 主键约束 | 否。 {.text-danger} | |
| E141-04 | 对于引用删除操作和引用更新操作，具有默认无操作的基本外键约束 | 否。 {.text-danger} | |
| E141-06 | 检查约束 | 是 {.text-success} | |
| E141-07 | 列默认值 | 是 {.text-success} | |
| E141-08 | 在主键上推断为非NULL | 是 {.text-success} | |
| E141-10 | 可以按任何顺序指定外键中的名称 | 非也。 {.text-danger} | |
| **E151** | **交易支持** | **非也。**{.text-danger} | |
| E151-01 | 提交语句 | 非也。 {.text-danger} | |
| E151-02 | 回滚语句 | 非也。 {.text-danger} | |
| **E152** | **基本设置事务语句** | **非也。**{.text-danger} | |
| E152-01 | SET TRANSACTION语句：隔离级别SERIALIZABLE子句 | 非也。 {.text-danger} | |
| E152-02 | SET TRANSACTION语句：只读和读写子句 | 非也。 {.text-danger} | |
| **E153** | **具有子查询的可更新查询** | **非也。**{.text-danger} | |
| E141-10 | 可以按任何顺序指定外键中的名称 | 否。 {.text-danger} | |
| **E151** | **交易支持** | **否。**{.text-danger} | |
| E151-01 | 提交语句 | 否。 {.text-danger} | |
| E151-02 | 回滚语句 | 否。 {.text-danger} | |
| **E152** | **基本设置事务语句** | **否。**{.text-danger} | |
| E152-01 | SET TRANSACTION语句：隔离级别SERIALIZABLE子句 | 否。 {.text-danger} | |
| E152-02 | SET TRANSACTION语句：只读和读写子句 | 否。 {.text-danger} | |
| **E153** | **具有子查询的可更新查询** | **否。**{.text-danger} | |
| **E161** | **SQL注释使用前导双减** | **是**{.text-success} | |
| **E171** | **SQLSTATE支持** | **非也。**{.text-danger} | |
| **E182** | **主机语言绑定** | **非也。**{.text-danger} | |
| **E171** | **SQLSTATE支持** | **否。**{.text-danger} | |
| **E182** | **主机语言绑定** | **否。**{.text-danger} | |
| **F031** | **基本架构操作** | **部分**{.text-warning} | |
| F031-01 | CREATE TABLE语句创建持久基表 | 部分 {.text-warning} | 非也。 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 子句，不支持用户解析的数据类型 |
| F031-02 | 创建视图语句 | 部分 {.text-warning} | 非也。 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 子句，不支持用户解析的数据类型 |
| F031-01 | CREATE TABLE语句创建持久基表 | 部分 {.text-warning} | 否。 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 子句，不支持用户解析的数据类型 |
| F031-02 | 创建视图语句 | 部分 {.text-warning} | 否。 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 子句，不支持用户解析的数据类型 |
| F031-03 | 赠款声明 | 是 {.text-success} | |
| F031-04 | ALTER TABLE语句：ADD COLUMN子句 | 部分 {.text-warning} | 不支持 `GENERATED` 条款和系统时间段 |
| F031-13 | DROP TABLE语句：RESTRICT子句 | 非也。 {.text-danger} | |
| F031-16 | DROP VIEW语句：RESTRICT子句 | 非也。 {.text-danger} | |
| F031-19 | REVOKE语句：RESTRICT子句 | 非也。 {.text-danger} | |
| F031-13 | DROP TABLE语句：RESTRICT子句 | 否。 {.text-danger} | |
| F031-16 | DROP VIEW语句：RESTRICT子句 | 否。 {.text-danger} | |
| F031-19 | REVOKE语句：RESTRICT子句 | 否。 {.text-danger} | |
| **F041** | **基本连接表** | **部分**{.text-warning} | |
| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | |
| F041-02 | 内部关键字 | 是 {.text-success} | |
@ -137,16 +137,16 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | |
| F041-05 | 可以嵌套外部连接 | 是 {.text-success} | |
| F041-07 | 左侧或右侧外部联接中的内部表也可用于内部联接 | 是 {.text-success} | |
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 非也。 {.text-danger} | |
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否。 {.text-danger} | |
| **F051** | **基本日期和时间** | **部分**{.text-warning} | |
| F051-01 | 日期数据类型（包括对日期文字的支持) | 部分 {.text-warning} | 没有文字 |
| F051-02 | 时间数据类型（包括对时间文字的支持），秒小数精度至少为0 | 非也。 {.text-danger} | |
| F051-03 | 时间戳数据类型（包括对时间戳文字的支持），小数秒精度至少为0和6 | 非也。 {.text-danger} | `DateTime64` 时间提供了类似的功能 |
| F051-02 | 时间数据类型（包括对时间文字的支持），秒小数精度至少为0 | 否。 {.text-danger} | |
| F051-03 | 时间戳数据类型（包括对时间戳文字的支持），小数秒精度至少为0和6 | 否。 {.text-danger} | `DateTime64` 时间提供了类似的功能 |
| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 部分 {.text-warning} | 只有一种数据类型可用 |
| F051-05 | Datetime类型和字符串类型之间的显式转换 | 是 {.text-success} | |
| F051-06 | CURRENT_DATE | 非也。 {.text-danger} | `today()` 是相似的 |
| F051-07 | LOCALTIME | 非也。 {.text-danger} | `now()` 是相似的 |
| F051-08 | LOCALTIMESTAMP | 非也。 {.text-danger} | |
| F051-06 | CURRENT_DATE | 否。 {.text-danger} | `today()` 是相似的 |
| F051-07 | LOCALTIME | 否。 {.text-danger} | `now()` 是相似的 |
| F051-08 | LOCALTIMESTAMP | 否。 {.text-danger} | |
| **F081** | **联盟和视图除外** | **部分**{.text-warning} | |
| **F131** | **分组操作** | **部分**{.text-warning} | |
| F131-01 | WHERE、GROUP BY和HAVING子句在具有分组视图的查询中受支持 | 是 {.text-success} | |
@ -154,27 +154,27 @@ toc_title: "ANSI\u517C\u5BB9\u6027"
| F131-03 | 设置具有分组视图的查询中支持的函数 | 是 {.text-success} | |
| F131-04 | 具有分组依据和具有子句和分组视图的子查询 | 是 {.text-success} | |
| F131-05 | 单行选择具有GROUP BY和具有子句和分组视图 | 非也。 {.text-danger} | |
| **F181** | **多模块支持** | **非也。**{.text-danger} | |
| **F181** | **多模块支持** | **否。**{.text-danger} | |
| **F201** | **投函数** | **是**{.text-success} | |
| **F221** | **显式默认值** | **非也。**{.text-danger} | |
| **F221** | **显式默认值** | **否。**{.text-danger} | |
| **F261** | **案例表达式** | **是**{.text-success} | |
| F261-01 | 简单案例 | 是 {.text-success} | |
| F261-02 | 检索案例 | 是 {.text-success} | |
| F261-03 | NULLIF | 是 {.text-success} | |
| F261-04 | COALESCE | 是 {.text-success} | |
| **F311** | **架构定义语句** | **部分**{.text-warning} | |
| F311-01 | CREATE SCHEMA | 非也。 {.text-danger} | |
| F311-01 | CREATE SCHEMA | 否。 {.text-danger} | |
| F311-02 | 为持久基表创建表 | 是 {.text-success} | |
| F311-03 | CREATE VIEW | 是 {.text-success} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 非也。 {.text-danger} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否。 {.text-danger} | |
| F311-05 | 赠款声明 | 是 {.text-success} | |
| **F471** | **标量子查询值** | **是**{.text-success} | |
| **F481** | **扩展空谓词** | **是**{.text-success} | |
| **F812** | **基本标记** | **非也。**{.text-danger} | |
| **T321** | **基本的SQL调用例程** | **非也。**{.text-danger} | |
| T321-01 | 无重载的用户定义函数 | 非也。 {.text-danger} | |
| T321-02 | 无重载的用户定义存储过程 | 非也。 {.text-danger} | |
| T321-03 | 函数调用 | 非也。 {.text-danger} | |
| T321-04 | 电话声明 | 非也。 {.text-danger} | |
| T321-05 | 退货声明 | 非也。 {.text-danger} | |
| **F812** | **基本标记** | **否。**{.text-danger} | |
| **T321** | **基本的SQL调用例程** | **否。**{.text-danger} | |
| T321-01 | 无重载的用户定义函数 | 否。 {.text-danger} | |
| T321-02 | 无重载的用户定义存储过程 | 否。 {.text-danger} | |
| T321-03 | 函数调用 | 否。 {.text-danger} | |
| T321-04 | 电话声明 | 否。 {.text-danger} | |
| T321-05 | 退货声明 | 否。 {.text-danger} | |
| **T631** | **在一个列表元素的谓词中** | **是**{.text-success} | |

450
programs/client/Client.cpp
View File

@ -195,15 +195,15 @@ private:
/// Parsed query. Is used to determine some settings (e.g. format, output file).
ASTPtr parsed_query;
/// The last exception that was received from the server. Is used for the return code in batch mode.
std::unique_ptr<Exception> last_exception_received_from_server;
/// The last exception that was received from the server. Is used for the
/// return code in batch mode.
std::unique_ptr<Exception> server_exception;
/// Likewise, the last exception that occurred on the client.
std::unique_ptr<Exception> client_exception;
/// If the last query resulted in exception.
bool received_exception_from_server = false;
int expected_server_error = 0;
int expected_client_error = 0;
int actual_server_error = 0;
int actual_client_error = 0;
/// If the last query resulted in exception. `server_exception` or
/// `client_exception` must be set.
bool have_error = false;
UInt64 server_revision = 0;
String server_version;
@ -669,19 +669,16 @@ private:
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
// We don't need to handle the test hints in the interactive
// mode.
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
std::cerr << std::endl;
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
@ -707,9 +704,20 @@ private:
nonInteractive();
/// If exception code isn't zero, we should return non-zero return code anyway.
if (last_exception_received_from_server)
return last_exception_received_from_server->code() != 0 ? last_exception_received_from_server->code() : -1;
// If exception code isn't zero, we should return non-zero return
// code anyway.
const auto * exception = server_exception
? server_exception.get() : client_exception.get();
if (exception)
{
return exception->code() != 0 ? exception->code() : -1;
}
if (have_error)
{
// Shouldn't be set without an exception, but check it just in
// case so that at least we don't lose an error.
return -1;
}
return 0;
}
@ -856,8 +864,93 @@ private:
return processMultiQuery(text);
}
// Consumes trailing semicolons and tries to consume the same-line trailing
// comment.
static void adjustQueryEnd(const char *& this_query_end,
const char * all_queries_end, int max_parser_depth)
{
// We have to skip the trailing semicolon that might be left
// after VALUES parsing or just after a normal semicolon-terminated query.
Tokens after_query_tokens(this_query_end, all_queries_end);
IParser::Pos after_query_iterator(after_query_tokens, max_parser_depth);
while (after_query_iterator.isValid()
&& after_query_iterator->type == TokenType::Semicolon)
{
this_query_end = after_query_iterator->end;
++after_query_iterator;
}
// Now we have to do some extra work to add the trailing
// same-line comment to the query, but preserve the leading
// comments of the next query. The trailing comment is important
// because the test hints are usually written this way, e.g.:
// select nonexistent_column; -- { serverError 12345 }.
// The token iterator skips comments and whitespace, so we have
// to find the newline in the string manually. If it's earlier
// than the next significant token, it means that the text before
// newline is some trailing whitespace or comment, and we should
// add it to our query. There are also several special cases
// that are described below.
const auto * newline = find_first_symbols<'\n'>(this_query_end,
all_queries_end);
const char * next_query_begin = after_query_iterator->begin;
// We include the entire line if the next query starts after
// it. This is a generic case of trailing in-line comment.
// The "equals" condition is for case of end of input (they both equal
// all_queries_end);
if (newline <= next_query_begin)
{
assert(newline >= this_query_end);
this_query_end = newline;
}
else
{
// Many queries on one line, can't do anything. By the way, this
// syntax is probably going to work as expected:
// select nonexistent /* { serverError 12345 } */; select 1
}
}
void reportQueryError() const
{
// If we probably have progress bar, we should add additional
// newline, otherwise exception may display concatenated with
// the progress bar.
if (need_render_progress)
std::cerr << '\n';
if (server_exception)
{
std::string text = server_exception->displayText();
auto embedded_stack_trace_pos = text.find("Stack trace");
if (std::string::npos != embedded_stack_trace_pos
&& !config().getBool("stacktrace", false))
{
text.resize(embedded_stack_trace_pos);
}
std::cerr << "Received exception from server (version "
<< server_version << "):" << std::endl << "Code: "
<< server_exception->code() << ". " << text << std::endl;
}
if (client_exception)
{
fmt::print(stderr,
"Error on processing query '{}':\n{}",
full_query, client_exception->message());
}
// A debug check -- at least some exception must be set, if the error
// flag is set, and vice versa.
assert(have_error == (client_exception || server_exception));
}
bool processMultiQuery(const String & all_queries_text)
{
// It makes sense not to base any control flow on this, so that it is
// the same in tests and in normal usage. The only difference is that in
// normal mode we ignore the test hints.
const bool test_mode = config().has("testmode");
{
@ -882,35 +975,31 @@ private:
while (this_query_begin < all_queries_end)
{
// Use the token iterator to skip any whitespace, semicolons and
// comments at the beginning of the query. An example from regression
// tests:
// insert into table t values ('invalid'); -- { serverError 469 }
// select 1
// Here the test hint comment gets parsed as a part of second query.
// We parse the `INSERT VALUES` up to the semicolon, and the rest
// looks like a two-line query:
// -- { serverError 469 }
// select 1
// and we expect it to fail with error 469, but this hint is actually
// for the previous query. Test hints should go after the query, so
// we can fix this by skipping leading comments. Token iterator skips
// comments and whitespace by itself, so we only have to check for
// semicolons.
// The code block is to limit visibility of `tokens` because we have
// another such variable further down the code, and get warnings for
// that.
// Remove leading empty newlines and other whitespace, because they
// are annoying to filter in query log. This is mostly relevant for
// the tests.
while (this_query_begin < all_queries_end
&& isWhitespaceASCII(*this_query_begin))
{
++this_query_begin;
}
if (this_query_begin >= all_queries_end)
{
break;
}
// If there are only comments left until the end of file, we just
// stop. The parser can't handle this situation because it always
// expects that there is some query that it can parse.
// We can get into this situation because the parser also doesn't
// skip the trailing comments after parsing a query. This is because
// they may as well be the leading comments for the next query,
// and it makes more sense to treat them as such.
{
Tokens tokens(this_query_begin, all_queries_end);
IParser::Pos token_iterator(tokens,
context.getSettingsRef().max_parser_depth);
while (token_iterator->type == TokenType::Semicolon
&& token_iterator.isValid())
{
++token_iterator;
}
this_query_begin = token_iterator->begin;
if (this_query_begin >= all_queries_end)
if (!token_iterator.isValid())
{
break;
}
@ -924,14 +1013,23 @@ private:
}
catch (Exception & e)
{
if (!test_mode)
throw;
// Try to find test hint for syntax error. We don't know where
// the query ends because we failed to parse it, so we consume
// the entire line.
this_query_end = find_first_symbols<'\n'>(this_query_end,
all_queries_end);
/// Try find test hint for syntax error
const char * end_of_line = find_first_symbols<'\n'>(this_query_begin,all_queries_end);
TestHint hint(true, String(this_query_end, end_of_line - this_query_end));
if (hint.serverError()) /// Syntax errors are considered as client errors
TestHint hint(test_mode,
String(this_query_begin, this_query_end - this_query_begin));
if (hint.serverError())
{
// Syntax errors are considered as client errors
e.addMessage("\nExpected server error '{}'.",
hint.serverError());
throw;
}
if (hint.clientError() != e.code())
{
if (hint.clientError())
@ -940,7 +1038,7 @@ private:
}
/// It's expected syntax error, skip the line
this_query_begin = end_of_line;
this_query_begin = this_query_end;
continue;
}
@ -968,10 +1066,14 @@ private:
// The VALUES format needs even more handling -- we also allow the
// data to be delimited by semicolon. This case is handled later by
// the format parser itself.
// We can't do multiline INSERTs with inline data, because most
// row input formats (e.g. TSV) can't tell when the input stops,
// unlike VALUES.
auto * insert_ast = parsed_query->as<ASTInsertQuery>();
if (insert_ast && insert_ast->data)
{
this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end);
this_query_end = find_first_symbols<'\n'>(insert_ast->data,
all_queries_end);
insert_ast->end = this_query_end;
query_to_send = all_queries_text.substr(
this_query_begin - all_queries_text.data(),
@ -984,60 +1086,158 @@ private:
this_query_end - this_query_begin);
}
// full_query is the query + inline INSERT data.
// Try to include the trailing comment with test hints. It is just
// a guess for now, because we don't yet know where the query ends
// if it is an INSERT query with inline data. We will do it again
// after we have processed the query. But even this guess is
// beneficial so that we see proper trailing comments in "echo" and
// server log.
adjustQueryEnd(this_query_end, all_queries_end,
context.getSettingsRef().max_parser_depth);
// full_query is the query + inline INSERT data + trailing comments
// (the latter is our best guess for now).
full_query = all_queries_text.substr(
this_query_begin - all_queries_text.data(),
this_query_end - this_query_begin);
// Look for the hint in the text of query + insert data, if any.
// e.g. insert into t format CSV 'a' -- { serverError 123 }.
TestHint test_hint(test_mode, full_query);
expected_client_error = test_hint.clientError();
expected_server_error = test_hint.serverError();
if (query_fuzzer_runs)
{
if (!processWithFuzzing(full_query))
return false;
this_query_begin = this_query_end;
continue;
}
try
{
processParsedSingleQuery();
}
catch (...)
{
// Surprisingly, this is a client error. A server error would
// have been reported w/o throwing (see onReceiveSeverException()).
client_exception = std::make_unique<Exception>(
getCurrentExceptionMessage(true), getCurrentExceptionCode());
have_error = true;
}
// For INSERTs with inline data: use the end of inline data as
// reported by the format parser (it is saved in sendData()).
// This allows us to handle queries like:
// insert into t values (1); select 1
// , where the inline data is delimited by semicolon and not by a
// newline.
if (insert_ast && insert_ast->data)
{
this_query_end = insert_ast->end;
adjustQueryEnd(this_query_end, all_queries_end,
context.getSettingsRef().max_parser_depth);
}
// Now we know for sure where the query ends.
// Look for the hint in the text of query + insert data + trailing
// comments,
// e.g. insert into t format CSV 'a' -- { serverError 123 }.
// Use the updated query boundaries we just calculated.
TestHint test_hint(test_mode, std::string(this_query_begin,
this_query_end - this_query_begin));
// Check whether the error (or its absence) matches the test hints
// (or their absence).
bool error_matches_hint = true;
if (have_error)
{
if (test_hint.serverError())
{
if (!server_exception)
{
error_matches_hint = false;
fmt::print(stderr,
"Expected server error code '{}' but got no server error.\n",
test_hint.serverError());
}
else if (server_exception->code() != test_hint.serverError())
{
error_matches_hint = false;
std::cerr << "Expected server error code: " <<
test_hint.serverError() << " but got: " <<
server_exception->code() << "." << std::endl;
}
}
if (test_hint.clientError())
{
if (!client_exception)
{
error_matches_hint = false;
fmt::print(stderr,
"Expected client error code '{}' but got no client error.\n",
test_hint.clientError());
}
else if (client_exception->code() != test_hint.clientError())
{
error_matches_hint = false;
fmt::print(stderr,
"Expected client error code '{}' but got '{}'.\n",
test_hint.clientError(),
client_exception->code());
}
}
if (!test_hint.clientError() && !test_hint.serverError())
{
// No error was expected but it still occurred. This is the
// default case w/o test hint, doesn't need additional
// diagnostics.
error_matches_hint = false;
}
}
else
{
try
if (test_hint.clientError())
{
processParsedSingleQuery();
if (insert_ast && insert_ast->data)
{
// For VALUES format: use the end of inline data as reported
// by the format parser (it is saved in sendData()). This
// allows us to handle queries like:
// insert into t values (1); select 1
//, where the inline data is delimited by semicolon and not
// by a newline.
this_query_end = parsed_query->as<ASTInsertQuery>()->end;
}
}
catch (...)
{
last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
actual_client_error = last_exception_received_from_server->code();
if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message();
received_exception_from_server = true;
fmt::print(stderr,
"The query succeeded but the client error '{}' was expected.\n",
test_hint.clientError());
error_matches_hint = false;
}
if (!test_hint.checkActual(
actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server))
if (test_hint.serverError())
{
connection->forceConnected(connection_parameters.timeouts);
fmt::print(stderr,
"The query succeeded but the server error '{}' was expected.\n",
test_hint.serverError());
error_matches_hint = false;
}
}
if (received_exception_from_server && !ignore_error)
// If the error is expected, force reconnect and ignore it.
if (have_error && error_matches_hint)
{
client_exception.reset();
server_exception.reset();
have_error = false;
connection->forceConnected(connection_parameters.timeouts);
}
// Report error.
if (have_error)
{
reportQueryError();
}
// Stop processing queries if needed.
if (have_error && !ignore_error)
{
if (is_interactive)
{
if (is_interactive)
break;
else
return false;
break;
}
else
{
return false;
}
}
@ -1146,15 +1346,20 @@ private:
// Some functions (e.g. protocol parsers) don't throw, but
// set last_exception instead, so we'll also do it here for
// uniformity.
last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
received_exception_from_server = true;
// Surprisingly, this is a client exception, because we get the
// server exception w/o throwing (see onReceiveException()).
client_exception = std::make_unique<Exception>(
getCurrentExceptionMessage(true), getCurrentExceptionCode());
have_error = true;
}
if (received_exception_from_server)
if (have_error)
{
const auto * exception = server_exception
? server_exception.get() : client_exception.get();
fmt::print(stderr, "Error on processing query '{}': {}\n",
ast_to_process->formatForErrorMessage(),
last_exception_received_from_server->message());
exception->message());
}
if (!connection->isConnected())
@ -1167,13 +1372,14 @@ private:
// The server is still alive so we're going to continue fuzzing.
// Determine what we're going to use as the starting AST.
if (received_exception_from_server)
if (have_error)
{
// Query completed with error, keep the previous starting AST.
// Also discard the exception that we now know to be non-fatal,
// so that it doesn't influence the exit code.
last_exception_received_from_server.reset(nullptr);
received_exception_from_server = false;
server_exception.reset();
client_exception.reset();
have_error = false;
}
else if (ast_to_process->formatForErrorMessage().size() > 500)
{
@ -1217,6 +1423,11 @@ private:
}
processParsedSingleQuery();
if (have_error)
{
reportQueryError();
}
}
// Parameters are in global variables:
@ -1227,8 +1438,9 @@ private:
void processParsedSingleQuery()
{
resetOutput();
last_exception_received_from_server.reset();
received_exception_from_server = false;
client_exception.reset();
server_exception.reset();
have_error = false;
if (echo_queries)
{
@ -1293,7 +1505,7 @@ private:
}
/// Do not change context (current DB, settings) in case of an exception.
if (!received_exception_from_server)
if (!have_error)
{
if (const auto * set_query = parsed_query->as<ASTSetQuery>())
{
@ -1704,8 +1916,7 @@ private:
return true;
case Protocol::Server::Exception:
onReceiveExceptionFromServer(*packet.exception);
last_exception_received_from_server = std::move(packet.exception);
onReceiveExceptionFromServer(std::move(packet.exception));
return false;
case Protocol::Server::Log:
@ -1737,8 +1948,7 @@ private:
return true;
case Protocol::Server::Exception:
onReceiveExceptionFromServer(*packet.exception);
last_exception_received_from_server = std::move(packet.exception);
onReceiveExceptionFromServer(std::move(packet.exception));
return false;
case Protocol::Server::Log:
@ -1771,8 +1981,7 @@ private:
return true;
case Protocol::Server::Exception:
onReceiveExceptionFromServer(*packet.exception);
last_exception_received_from_server = std::move(packet.exception);
onReceiveExceptionFromServer(std::move(packet.exception));
return false;
case Protocol::Server::Log:
@ -2081,32 +2290,11 @@ private:
}
void onReceiveExceptionFromServer(const Exception & e)
void onReceiveExceptionFromServer(std::unique_ptr<Exception> && e)
{
have_error = true;
server_exception = std::move(e);
resetOutput();
received_exception_from_server = true;
actual_server_error = e.code();
if (expected_server_error)
{
if (actual_server_error == expected_server_error)
return;
std::cerr << "Expected error code: " << expected_server_error << " but got: " << actual_server_error << "." << std::endl;
}
std::string text = e.displayText();
auto embedded_stack_trace_pos = text.find("Stack trace");
if (std::string::npos != embedded_stack_trace_pos && !config().getBool("stacktrace", false))
text.resize(embedded_stack_trace_pos);
/// If we probably have progress bar, we should add additional newline,
/// otherwise exception may display concatenated with the progress bar.
if (need_render_progress)
std::cerr << '\n';
std::cerr << "Received exception from server (version " << server_version << "):" << std::endl
<< "Code: " << e.code() << ". " << text << std::endl;
}

8
programs/client/QueryFuzzer.cpp
View File

@ -22,6 +22,7 @@
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTUseQuery.h>
#include <Parsers/ASTWindowDefinition.h>
#include <Parsers/ParserQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
@ -403,10 +404,11 @@ void QueryFuzzer::fuzz(ASTPtr & ast)
fuzzColumnLikeExpressionList(fn->arguments.get());
fuzzColumnLikeExpressionList(fn->parameters.get());
if (fn->is_window_function)
if (fn->is_window_function && fn->window_definition)
{
fuzzColumnLikeExpressionList(fn->window_partition_by.get());
fuzzOrderByList(fn->window_order_by.get());
auto & def = fn->window_definition->as<ASTWindowDefinition &>();
fuzzColumnLikeExpressionList(def.partition_by.get());
fuzzOrderByList(def.order_by.get());
}
fuzz(fn->children);

69
programs/client/TestHint.h
View File

@ -11,30 +11,32 @@
namespace DB
{
namespace ErrorCodes
{
extern const int UNEXPECTED_ERROR_CODE;
}
/// Checks expected server and client error codes in testmode.
/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }".
/// Also you can enable echoing all queries by writing "-- { echo }".
class TestHint
{
public:
TestHint(bool enabled_, const String & query_)
: enabled(enabled_)
, query(query_)
TestHint(bool enabled_, const String & query_) :
query(query_)
{
if (!enabled_)
return;
// Don't parse error hints in leading comments, because it feels weird.
// Leading 'echo' hint is OK.
bool is_leading_hint = true;
Lexer lexer(query.data(), query.data() + query.size());
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
{
if (token.type == TokenType::Comment)
if (token.type != TokenType::Comment
&& token.type != TokenType::Whitespace)
{
is_leading_hint = false;
}
else if (token.type == TokenType::Comment)
{
String comment(token.begin, token.begin + token.size());
@ -47,7 +49,7 @@ public:
if (pos_end != String::npos)
{
String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end);
parse(hint);
parse(hint, is_leading_hint);
}
}
}
@ -55,46 +57,17 @@ public:
}
}
/// @returns true if it's possible to continue without reconnect
bool checkActual(int & actual_server_error, int & actual_client_error,
bool & got_exception, std::unique_ptr<Exception> & last_exception) const
{
if (!enabled)
return true;
if (allErrorsExpected(actual_server_error, actual_client_error))
{
got_exception = false;
last_exception.reset();
actual_server_error = 0;
actual_client_error = 0;
return false;
}
if (lostExpectedError(actual_server_error, actual_client_error))
{
std::cerr << "Success when error expected in query: " << query << "It expects server error "
<< server_error << ", client error " << client_error << "." << std::endl;
got_exception = true;
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::UNEXPECTED_ERROR_CODE); /// return error to OS
return false;
}
return true;
}
int serverError() const { return server_error; }
int clientError() const { return client_error; }
bool echoQueries() const { return echo; }
private:
bool enabled = false;
const String & query;
int server_error = 0;
int client_error = 0;
bool echo = false;
void parse(const String & hint)
void parse(const String & hint, bool is_leading_hint)
{
std::stringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
ss << hint;
@ -106,11 +79,15 @@ private:
if (ss.eof())
break;
if (item == "serverError")
ss >> server_error;
else if (item == "clientError")
ss >> client_error;
else if (item == "echo")
if (!is_leading_hint)
{
if (item == "serverError")
ss >> server_error;
else if (item == "clientError")
ss >> client_error;
}
if (item == "echo")
echo = true;
}
}

6
src/Core/MySQL/MySQLGtid.cpp
View File

@ -85,8 +85,8 @@ void GTIDSets::update(const GTID & other)
ErrorCodes::LOGICAL_ERROR);
}
/// Try to shirnk Sequence interval.
GTIDSet::tryShirnk(set, i, current);
/// Try to shrink Sequence interval.
GTIDSet::tryShrink(set, i, current);
/// Sequence, extend the interval.
if (other.seq_no == current.end)
@ -119,7 +119,7 @@ void GTIDSets::update(const GTID & other)
sets.emplace_back(set);
}
void GTIDSet::tryShirnk(GTIDSet & set, unsigned int i, GTIDSet::Interval & current)
void GTIDSet::tryShrink(GTIDSet & set, unsigned int i, GTIDSet::Interval & current)
{
if (i != set.intervals.size() -1)
{

2
src/Core/MySQL/MySQLGtid.h
View File

@ -27,7 +27,7 @@ public:
void tryMerge(size_t i);
static void tryShirnk(GTIDSet & set, unsigned int i, Interval & current);
static void tryShrink(GTIDSet & set, unsigned int i, Interval & current);
};
class GTIDSets

4
src/Core/tests/mysql_protocol.cpp
View File

@ -262,12 +262,12 @@ int main(int argc, char ** argv)
"20662d71-9d91-11ea-bbc2-0242ac110003:9",
"10662d71-9d91-11ea-bbc2-0242ac110003:6-7,20662d71-9d91-11ea-bbc2-0242ac110003:9"},
{"shirnk-sequence",
{"shrink-sequence",
"10662d71-9d91-11ea-bbc2-0242ac110003:1-3:4-5:7",
"10662d71-9d91-11ea-bbc2-0242ac110003:6",
"10662d71-9d91-11ea-bbc2-0242ac110003:1-7"},
{"shirnk-sequence",
{"shrink-sequence",
"10662d71-9d91-11ea-bbc2-0242ac110003:1-3:4-5:10",
"10662d71-9d91-11ea-bbc2-0242ac110003:8",
"10662d71-9d91-11ea-bbc2-0242ac110003:1-5:8:10"

10
src/DataTypes/DataTypeMap.cpp
View File

@ -65,6 +65,16 @@ static IColumn & extractNestedColumn(IColumn & column)
return assert_cast<ColumnMap &>(column).getNestedColumn();
}
DataTypePtr DataTypeMap::tryGetSubcolumnType(const String & subcolumn_name) const
{
return nested->tryGetSubcolumnType(subcolumn_name);
}
ColumnPtr DataTypeMap::getSubcolumn(const String & subcolumn_name, const IColumn & column) const
{
return nested->getSubcolumn(subcolumn_name, extractNestedColumn(column));
}
void DataTypeMap::serializeBinary(const Field & field, WriteBuffer & ostr) const
{
const auto & map = get<const Map &>(field);

4
src/DataTypes/DataTypeMap.h
View File

@ -32,6 +32,9 @@ public:
bool canBeInsideNullable() const override { return false; }
DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const override;
ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const override;
void serializeBinary(const Field & field, WriteBuffer & ostr) const override;
void deserializeBinary(Field & field, ReadBuffer & istr) const override;
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
@ -45,7 +48,6 @@ public:
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void enumerateStreamsImpl(const StreamCallback & callback, SubstreamPath & path) const override;
void serializeBinaryBulkStatePrefixImpl(

9
src/Databases/MySQL/MaterializeMySQLSyncThread.cpp
View File

@ -95,19 +95,22 @@ static void checkMySQLVariables(const mysqlxx::Pool::Entry & connection)
"(Variable_name = 'log_bin' AND upper(Value) = 'ON') "
"OR (Variable_name = 'binlog_format' AND upper(Value) = 'ROW') "
"OR (Variable_name = 'binlog_row_image' AND upper(Value) = 'FULL') "
"OR (Variable_name = 'default_authentication_plugin' AND upper(Value) = 'MYSQL_NATIVE_PASSWORD');";
"OR (Variable_name = 'default_authentication_plugin' AND upper(Value) = 'MYSQL_NATIVE_PASSWORD') "
"OR (Variable_name = 'log_bin_use_v1_row_events' AND upper(Value) = 'OFF');";
MySQLBlockInputStream variables_input(connection, check_query, variables_header, DEFAULT_BLOCK_SIZE, false, true);
Block variables_block = variables_input.read();
if (!variables_block || variables_block.rows() != 4)
if (!variables_block || variables_block.rows() != 5)
{
std::unordered_map<String, String> variables_error_message{
{"log_bin", "log_bin = 'ON'"},
{"binlog_format", "binlog_format='ROW'"},
{"binlog_row_image", "binlog_row_image='FULL'"},
{"default_authentication_plugin", "default_authentication_plugin='mysql_native_password'"}
{"default_authentication_plugin", "default_authentication_plugin='mysql_native_password'"},
{"log_bin_use_v1_row_events", "log_bin_use_v1_row_events='OFF'"}
};
ColumnPtr variable_name_column = variables_block.getByName("Variable_name").column;
for (size_t index = 0; index < variables_block.rows(); ++index)

1
src/IO/Operators.h
View File

@ -46,6 +46,7 @@ template <typename T> WriteBuffer & operator<< (WriteBuffer & buf, const T &
/// If you do not use the manipulators, the string is displayed without an escape, as is.
template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const String & x) { writeString(x, buf); return buf; }
template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const std::string_view & x) { writeString(StringRef(x), buf); return buf; }
template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const StringRef & x) { writeString(x, buf); return buf; }
template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const char & x) { writeChar(x, buf); return buf; }
template <> inline WriteBuffer & operator<< (WriteBuffer & buf, const pcg32_fast & x) { PcgSerializer::serializePcg32(x, buf); return buf; }

8
src/Interpreters/ActionsVisitor.cpp
View File

@ -738,13 +738,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
if (node.is_window_function)
{
// Also add columns from PARTITION BY and ORDER BY of window functions.
if (node.window_partition_by)
if (node.window_definition)
{
visit(node.window_partition_by, data);
}
if (node.window_order_by)
{
visit(node.window_order_by, data);
visit(node.window_definition, data);
}
// Also manually add columns for arguments of the window function itself.

3
src/Interpreters/ExpressionActions.h
View File

@ -79,6 +79,7 @@ private:
Block sample_block;
public:
ExpressionActions() = delete;
~ExpressionActions();
explicit ExpressionActions(ActionsDAGPtr actions_dag_);
ExpressionActions(const ExpressionActions &) = default;
@ -114,8 +115,6 @@ public:
ExpressionActionsPtr clone() const;
private:
ExpressionActions() = default;
void checkLimits(const ColumnsWithTypeAndName & columns) const;
void linearizeActions();

194
src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -1,12 +1,13 @@
#include <Core/Block.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTWindowDefinition.h>
#include <Parsers/DumpASTNode.h>
#include <DataTypes/DataTypeNullable.h>
@ -290,8 +291,6 @@ void ExpressionAnalyzer::analyzeAggregation()
{
aggregated_columns = temp_actions->getNamesAndTypesList();
}
has_window = makeWindowDescriptions(temp_actions);
}
@ -473,8 +472,52 @@ bool ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions)
return !aggregates().empty();
}
void makeWindowDescriptionFromAST(WindowDescription & desc, const IAST * ast)
{
const auto & definition = ast->as<const ASTWindowDefinition &>();
bool ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr & actions)
if (definition.partition_by)
{
for (const auto & column_ast : definition.partition_by->children)
{
const auto * with_alias = dynamic_cast<const ASTWithAlias *>(
column_ast.get());
if (!with_alias)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Expected a column in PARTITION BY in window definition,"
" got '{}'",
column_ast->formatForErrorMessage());
}
desc.partition_by.push_back(SortColumnDescription(
with_alias->getColumnName(), 1 /* direction */,
1 /* nulls_direction */));
}
}
if (definition.order_by)
{
for (const auto & column_ast
: definition.order_by->children)
{
// Parser should have checked that we have a proper element here.
const auto & order_by_element
= column_ast->as<ASTOrderByElement &>();
// Ignore collation for now.
desc.order_by.push_back(
SortColumnDescription(
order_by_element.children.front()->getColumnName(),
order_by_element.direction,
order_by_element.nulls_direction));
}
}
desc.full_sort_description = desc.partition_by;
desc.full_sort_description.insert(desc.full_sort_description.end(),
desc.order_by.begin(), desc.order_by.end());
}
void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
{
// Convenient to check here because at least we have the Context.
if (!syntax->window_function_asts.empty() &&
@ -485,57 +528,34 @@ bool ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr & actions)
syntax->window_function_asts[0]->formatForErrorMessage());
}
// Window definitions from the WINDOW clause
const auto * select_query = query->as<ASTSelectQuery>();
if (select_query && select_query->window())
{
for (const auto & ptr : select_query->window()->children)
{
const auto & elem = ptr->as<const ASTWindowListElement &>();
WindowDescription desc;
desc.window_name = elem.name;
makeWindowDescriptionFromAST(desc, elem.definition.get());
auto [it, inserted] = window_descriptions.insert(
{desc.window_name, desc});
if (!inserted)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Window '{}' is defined twice in the WINDOW clause",
desc.window_name);
}
}
}
// Window functions
for (const ASTFunction * function_node : syntax->window_function_asts)
{
assert(function_node->is_window_function);
WindowDescription window_description;
window_description.window_name = function_node->getWindowDescription();
if (function_node->window_partition_by)
{
for (const auto & column_ast
: function_node->window_partition_by->children)
{
const auto * with_alias = dynamic_cast<const ASTWithAlias *>(
column_ast.get());
if (!with_alias)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Expected a column in PARTITION BY for window '{}',"
" got '{}'", window_description.window_name,
column_ast->formatForErrorMessage());
}
window_description.partition_by.push_back(
SortColumnDescription(
with_alias->getColumnName(), 1 /* direction */,
1 /* nulls_direction */));
}
}
if (function_node->window_order_by)
{
for (const auto & column_ast
: function_node->window_order_by->children)
{
// Parser should have checked that we have a proper element here.
const auto & order_by_element
= column_ast->as<ASTOrderByElement &>();
// Ignore collation for now.
window_description.order_by.push_back(
SortColumnDescription(
order_by_element.children.front()->getColumnName(),
order_by_element.direction,
order_by_element.nulls_direction));
}
}
window_description.full_sort_description = window_description.partition_by;
window_description.full_sort_description.insert(
window_description.full_sort_description.end(),
window_description.order_by.begin(),
window_description.order_by.end());
WindowFunctionDescription window_function;
window_function.function_node = function_node;
window_function.column_name
@ -580,19 +600,43 @@ bool ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr & actions)
window_function.argument_types,
window_function.function_parameters, properties);
auto [it, inserted] = window_descriptions.insert(
{window_description.window_name, window_description});
if (!inserted)
// Find the window corresponding to this function. It may be either
// referenced by name and previously defined in WINDOW clause, or it
// may be defined inline.
if (!function_node->window_name.empty())
{
assert(it->second.full_sort_description
== window_description.full_sort_description);
auto it = window_descriptions.find(function_node->window_name);
if (it == std::end(window_descriptions))
{
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
"Window '{}' is not defined (referenced by '{}')",
function_node->window_name,
function_node->formatForErrorMessage());
}
it->second.window_functions.push_back(window_function);
}
else
{
const auto & definition = function_node->window_definition->as<
const ASTWindowDefinition &>();
WindowDescription desc;
desc.window_name = definition.getDefaultWindowName();
makeWindowDescriptionFromAST(desc, &definition);
it->second.window_functions.push_back(window_function);
auto [it, inserted] = window_descriptions.insert(
{desc.window_name, desc});
if (!inserted)
{
assert(it->second.full_sort_description
== desc.full_sort_description);
}
it->second.window_functions.push_back(window_function);
}
}
return !syntax->window_function_asts.empty();
}
@ -971,28 +1015,42 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
{
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
// 1) Add actions for window functions and their arguments;
// 2) Mark the columns that are really required. We have to mark them as
// required because we finish the expression chain before processing the
// window functions.
// (1) Add actions for window functions and the columns they require.
// (2) Mark the columns that are really required. We have to mark them as
// required because we finish the expression chain before processing the
// window functions.
// The required columns are:
// (a) window function arguments,
// (b) the columns from PARTITION BY and ORDER BY.
// (1a) Actions for PARTITION BY and ORDER BY for windows defined in the
// WINDOW clause. The inline window definitions will be processed
// recursively together with (1b) as ASTFunction::window_definition.
if (getSelectQuery()->window())
{
getRootActionsNoMakeSet(getSelectQuery()->window(),
true /* no_subqueries */, step.actions());
}
for (const auto & [_, w] : window_descriptions)
{
for (const auto & f : w.window_functions)
{
// 1.1) arguments of window functions;
// (1b) Actions for function arguments, and also the inline window
// definitions (1a).
// Requiring a constant reference to a shared pointer to non-const AST
// doesn't really look sane, but the visitor does indeed require it.
getRootActionsNoMakeSet(f.function_node->clone(),
true /* no_subqueries */, step.actions());
// 2.1) function arguments;
// (2b) Required function argument columns.
for (const auto & a : f.function_node->arguments->children)
{
step.required_output.push_back(a->getColumnName());
}
}
// 2.1) PARTITION BY and ORDER BY columns.
// (2a) Required PARTITION BY and ORDER BY columns.
for (const auto & c : w.full_sort_description)
{
step.required_output.push_back(c.column_name);
@ -1411,6 +1469,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
// the main SELECT, similar to what we do for aggregate functions.
if (has_window)
{
query_analyzer.makeWindowDescriptions(chain.getLastActions());
query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage);
// Build a list of output columns of the window step.

7
src/Interpreters/ExpressionAnalyzer.h
View File

@ -62,7 +62,6 @@ struct ExpressionAnalyzerData
NamesAndTypesList aggregation_keys;
AggregateDescriptions aggregate_descriptions;
bool has_window = false;
WindowDescriptions window_descriptions;
NamesAndTypesList window_columns;
@ -125,6 +124,8 @@ public:
/// A list of windows for window functions.
const WindowDescriptions & windowDescriptions() const { return window_descriptions; }
void makeWindowDescriptions(ActionsDAGPtr actions);
protected:
ExpressionAnalyzer(
const ASTPtr & query_,
@ -168,8 +169,6 @@ protected:
void analyzeAggregation();
bool makeAggregateDescriptions(ActionsDAGPtr & actions);
bool makeWindowDescriptions(ActionsDAGPtr & actions);
const ASTSelectQuery * getSelectQuery() const;
bool isRemoteStorage() const { return syntax->is_remote_storage; }
@ -272,7 +271,7 @@ public:
/// Does the expression have aggregate functions or a GROUP BY or HAVING section.
bool hasAggregation() const { return has_aggregation; }
bool hasWindow() const { return has_window; }
bool hasWindow() const { return !syntax->window_function_asts.empty(); }
bool hasGlobalSubqueries() { return has_global_subqueries; }
bool hasTableJoin() const { return syntax->ast_join; }

9
src/Interpreters/QueryNormalizer.cpp
View File

@ -177,14 +177,9 @@ void QueryNormalizer::visitChildren(IAST * node, Data & data)
}
}
if (func_node->window_partition_by)
if (func_node->window_definition)
{
visitChildren(func_node->window_partition_by.get(), data);
}
if (func_node->window_order_by)
{
visitChildren(func_node->window_order_by.get(), data);
visitChildren(func_node->window_definition.get(), data);
}
}
else if (!node->as<ASTSelectQuery>())

17
src/Interpreters/TreeRewriter.cpp
View File

@ -466,6 +466,8 @@ std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSel
assertNoWindows(select_query.where(), "in WHERE");
if (select_query.prewhere())
assertNoWindows(select_query.prewhere(), "in PREWHERE");
if (select_query.window())
assertNoWindows(select_query.window(), "in WINDOW");
GetAggregatesVisitor::Data data;
GetAggregatesVisitor(data).visit(query);
@ -483,20 +485,9 @@ std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSel
}
}
if (node->window_partition_by)
if (node->window_definition)
{
for (auto & arg : node->window_partition_by->children)
{
assertNoWindows(arg, "inside PARTITION BY of a window");
}
}
if (node->window_order_by)
{
for (auto & arg : node->window_order_by->children)
{
assertNoWindows(arg, "inside ORDER BY of a window");
}
assertNoWindows(node->window_definition, "inside window definition");
}
}

81
src/Parsers/ASTFunction.cpp
View File

@ -1,5 +1,6 @@
#include <Parsers/ASTFunction.h>
#include <Common/quoteString.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
#include <IO/Operators.h>
@ -42,12 +43,20 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
if (is_window_function)
{
writeCString(" OVER (", ostr);
FormatSettings settings{ostr, true /* one_line */};
FormatState state;
FormatStateStacked frame;
appendWindowDescription(settings, state, frame);
writeCString(")", ostr);
writeCString(" OVER ", ostr);
if (!window_name.empty())
{
ostr << window_name;
}
else
{
FormatSettings settings{ostr, true /* one_line */};
FormatState state;
FormatStateStacked frame;
writeCString("(", ostr);
window_definition->formatImpl(settings, state, frame);
writeCString(")", ostr);
}
}
}
@ -65,22 +74,10 @@ ASTPtr ASTFunction::clone() const
if (arguments) { res->arguments = arguments->clone(); res->children.push_back(res->arguments); }
if (parameters) { res->parameters = parameters->clone(); res->children.push_back(res->parameters); }
if (window_name)
if (window_definition)
{
res->window_name = window_name->clone();
res->children.push_back(res->window_name);
}
if (window_partition_by)
{
res->window_partition_by = window_partition_by->clone();
res->children.push_back(res->window_partition_by);
}
if (window_order_by)
{
res->window_order_by = window_order_by->clone();
res->children.push_back(res->window_order_by);
res->window_definition = window_definition->clone();
res->children.push_back(res->window_definition);
}
return res;
@ -487,44 +484,16 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
return;
}
settings.ostr << " OVER (";
appendWindowDescription(settings, state, nested_dont_need_parens);
settings.ostr << ")";
}
std::string ASTFunction::getWindowDescription() const
{
WriteBufferFromOwnString ostr;
FormatSettings settings{ostr, true /* one_line */};
FormatState state;
FormatStateStacked frame;
appendWindowDescription(settings, state, frame);
return ostr.str();
}
void ASTFunction::appendWindowDescription(const FormatSettings & settings,
FormatState & state, FormatStateStacked frame) const
{
if (!is_window_function)
settings.ostr << " OVER ";
if (!window_name.empty())
{
return;
settings.ostr << backQuoteIfNeed(window_name);
}
if (window_partition_by)
else
{
settings.ostr << "PARTITION BY ";
window_partition_by->formatImpl(settings, state, frame);
}
if (window_partition_by && window_order_by)
{
settings.ostr << " ";
}
if (window_order_by)
{
settings.ostr << "ORDER BY ";
window_order_by->formatImpl(settings, state, frame);
settings.ostr << "(";
window_definition->formatImpl(settings, state, frame);
settings.ostr << ")";
}
}

13
src/Parsers/ASTFunction.h
View File

@ -32,14 +32,8 @@ public:
// pointers of proper type (see e.g. IAST::set), but this is not compatible
// with the visitor interface.
// ASTIdentifier
ASTPtr window_name;
// ASTExpressionList
ASTPtr window_partition_by;
// ASTExpressionList of
ASTPtr window_order_by;
String window_name;
ASTPtr window_definition;
/// do not print empty parentheses if there are no args - compatibility with new AST for data types and engine names.
bool no_empty_args = false;
@ -55,9 +49,6 @@ public:
ASTPtr toLiteral() const; // Try to convert functions like Array or Tuple to a literal form.
void appendWindowDescription(const FormatSettings & settings,
FormatState & state, FormatStateStacked frame) const;
std::string getWindowDescription() const;
protected:

8
src/Parsers/ASTSelectQuery.cpp
View File

@ -44,6 +44,7 @@ ASTPtr ASTSelectQuery::clone() const
CLONE(Expression::WHERE);
CLONE(Expression::GROUP_BY);
CLONE(Expression::HAVING);
CLONE(Expression::WINDOW);
CLONE(Expression::ORDER_BY);
CLONE(Expression::LIMIT_BY_OFFSET);
CLONE(Expression::LIMIT_BY_LENGTH);
@ -133,6 +134,13 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F
having()->formatImpl(s, state, frame);
}
if (window())
{
s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str <<
"WINDOW " << (s.hilite ? hilite_none : "");
window()->formatImpl(s, state, frame);
}
if (orderBy())
{
s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY" << (s.hilite ? hilite_none : "");

2
src/Parsers/ASTSelectQuery.h
View File

@ -25,6 +25,7 @@ public:
WHERE,
GROUP_BY,
HAVING,
WINDOW,
ORDER_BY,
LIMIT_BY_OFFSET,
LIMIT_BY_LENGTH,
@ -58,6 +59,7 @@ public:
const ASTPtr where() const { return getExpression(Expression::WHERE); }
const ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); }
const ASTPtr having() const { return getExpression(Expression::HAVING); }
const ASTPtr window() const { return getExpression(Expression::WINDOW); }
const ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); }
const ASTPtr limitByOffset() const { return getExpression(Expression::LIMIT_BY_OFFSET); }
const ASTPtr limitByLength() const { return getExpression(Expression::LIMIT_BY_LENGTH); }

89
src/Parsers/ASTWindowDefinition.cpp Normal file
View File

@ -0,0 +1,89 @@
#include <Parsers/ASTWindowDefinition.h>
#include <Common/quoteString.h>
#include <IO/Operators.h>
namespace DB
{
ASTPtr ASTWindowDefinition::clone() const
{
auto result = std::make_shared<ASTWindowDefinition>();
if (partition_by)
{
result->partition_by = partition_by->clone();
result->children.push_back(result->partition_by);
}
if (order_by)
{
result->order_by = order_by->clone();
result->children.push_back(result->order_by);
}
return result;
}
String ASTWindowDefinition::getID(char) const
{
return "WindowDefinition";
}
void ASTWindowDefinition::formatImpl(const FormatSettings & settings,
FormatState & state, FormatStateStacked frame) const
{
if (partition_by)
{
settings.ostr << "PARTITION BY ";
partition_by->formatImpl(settings, state, frame);
}
if (partition_by && order_by)
{
settings.ostr << " ";
}
if (order_by)
{
settings.ostr << "ORDER BY ";
order_by->formatImpl(settings, state, frame);
}
}
std::string ASTWindowDefinition::getDefaultWindowName() const
{
WriteBufferFromOwnString ostr;
FormatSettings settings{ostr, true /* one_line */};
FormatState state;
FormatStateStacked frame;
formatImpl(settings, state, frame);
return ostr.str();
}
ASTPtr ASTWindowListElement::clone() const
{
auto result = std::make_shared<ASTWindowListElement>();
result->name = name;
result->definition = definition->clone();
result->children.push_back(result->definition);
return result;
}
String ASTWindowListElement::getID(char) const
{
return "WindowListElement";
}
void ASTWindowListElement::formatImpl(const FormatSettings & settings,
FormatState & state, FormatStateStacked frame) const
{
settings.ostr << backQuoteIfNeed(name);
settings.ostr << " AS (";
definition->formatImpl(settings, state, frame);
settings.ostr << ")";
}
}

40
src/Parsers/ASTWindowDefinition.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#include <Parsers/IAST.h>
namespace DB
{
struct ASTWindowDefinition : public IAST
{
ASTPtr partition_by;
ASTPtr order_by;
ASTPtr clone() const override;
String getID(char delimiter) const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
std::string getDefaultWindowName() const;
};
struct ASTWindowListElement : public IAST
{
String name;
// ASTWindowDefinition
ASTPtr definition;
ASTPtr clone() const override;
String getID(char delimiter) const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
}

158
src/Parsers/ExpressionElementParsers.cpp
View File

@ -8,21 +8,22 @@
#include <Common/typeid_cast.h>
#include <Parsers/DumpASTNode.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTAsterisk.h>
#include <Parsers/ASTColumnsTransformers.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTAsterisk.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/ASTQueryParameter.h>
#include <Parsers/ASTTTLElement.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
#include <Parsers/ASTColumnsTransformers.h>
#include <Parsers/ASTTTLElement.h>
#include <Parsers/ASTWindowDefinition.h>
#include <Parsers/IAST.h>
#include <Parsers/parseIdentifierOrStringLiteral.h>
#include <Parsers/parseIntervalKind.h>
@ -462,8 +463,8 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
// of a different type, hence this workaround with a temporary pointer.
ASTPtr function_node_as_iast = function_node;
ParserWindowDefinition window_definition;
if (!window_definition.parse(pos, function_node_as_iast, expected))
ParserWindowReference window_reference;
if (!window_reference.parse(pos, function_node_as_iast, expected))
{
return false;
}
@ -473,7 +474,7 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
return true;
}
bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool ParserWindowReference::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ASTFunction * function = dynamic_cast<ASTFunction *>(node.get());
@ -488,8 +489,7 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
ParserIdentifier window_name_parser;
if (window_name_parser.parse(pos, window_name_ast, expected))
{
function->children.push_back(window_name_ast);
function->window_name = window_name_ast;
function->window_name = getIdentifierName(window_name_ast);
return true;
}
else
@ -497,10 +497,23 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
return false;
}
}
++pos;
// Variant 2:
// function_name ( * ) OVER ( window_definition )
ParserWindowDefinition parser_definition;
return parser_definition.parse(pos, function->window_definition, expected);
}
bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
auto result = std::make_shared<ASTWindowDefinition>();
ParserToken parser_openging_bracket(TokenType::OpeningRoundBracket);
if (!parser_openging_bracket.ignore(pos, expected))
{
return false;
}
ParserKeyword keyword_partition_by("PARTITION BY");
ParserNotEmptyExpressionList columns_partition_by(
false /* we don't allow declaring aliases here*/);
@ -512,8 +525,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
ASTPtr partition_by_ast;
if (columns_partition_by.parse(pos, partition_by_ast, expected))
{
function->children.push_back(partition_by_ast);
function->window_partition_by = partition_by_ast;
result->children.push_back(partition_by_ast);
result->partition_by = partition_by_ast;
}
else
{
@ -526,8 +539,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
ASTPtr order_by_ast;
if (columns_order_by.parse(pos, order_by_ast, expected))
{
function->children.push_back(order_by_ast);
function->window_order_by = order_by_ast;
result->children.push_back(order_by_ast);
result->order_by = order_by_ast;
}
else
{
@ -535,13 +548,55 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
}
}
if (pos->type != TokenType::ClosingRoundBracket)
ParserToken parser_closing_bracket(TokenType::ClosingRoundBracket);
if (!parser_closing_bracket.ignore(pos, expected))
{
expected.add(pos, "')'");
return false;
}
++pos;
node = result;
return true;
}
bool ParserWindowList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
auto result = std::make_shared<ASTExpressionList>();
for (;;)
{
auto elem = std::make_shared<ASTWindowListElement>();
ParserIdentifier parser_window_name;
ASTPtr window_name_identifier;
if (!parser_window_name.parse(pos, window_name_identifier, expected))
{
return false;
}
elem->name = getIdentifierName(window_name_identifier);
ParserKeyword keyword_as("AS");
if (!keyword_as.ignore(pos, expected))
{
return false;
}
ParserWindowDefinition parser_window_definition;
if (!parser_window_definition.parse(pos, elem->definition, expected))
{
return false;
}
result->children.push_back(elem);
// If the list countinues, there should be a comma.
ParserToken parser_comma(TokenType::Comma);
if (!parser_comma.ignore(pos))
{
break;
}
}
node = result;
return true;
}
@ -1316,41 +1371,42 @@ bool ParserLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
const char * ParserAlias::restricted_keywords[] =
{
"FROM",
"FINAL",
"SAMPLE",
"ARRAY",
"LEFT",
"RIGHT",
"INNER",
"FULL",
"CROSS",
"JOIN",
"GLOBAL",
"ANY",
"ALL",
"ASOF",
"SEMI",
"ANTI",
"ONLY", /// YQL synonym for ANTI. Note: YQL is the name of one of Yandex proprietary languages, completely unrelated to ClickHouse.
"ON",
"USING",
"PREWHERE",
"WHERE",
"GROUP",
"WITH",
"HAVING",
"ORDER",
"LIMIT",
"OFFSET",
"SETTINGS",
"FORMAT",
"UNION",
"INTO",
"NOT",
"ANY",
"ARRAY",
"ASOF",
"BETWEEN",
"LIKE",
"CROSS",
"FINAL",
"FORMAT",
"FROM",
"FULL",
"GLOBAL",
"GROUP",
"HAVING",
"ILIKE",
"INNER",
"INTO",
"JOIN",
"LEFT",
"LIKE",
"LIMIT",
"NOT",
"OFFSET",
"ON",
"ONLY", /// YQL synonym for ANTI. Note: YQL is the name of one of Yandex proprietary languages, completely unrelated to ClickHouse.
"ORDER",
"PREWHERE",
"RIGHT",
"SAMPLE",
"SEMI",
"SETTINGS",
"UNION",
"USING",
"WHERE",
"WINDOW",
"WITH",
nullptr
};

17
src/Parsers/ExpressionElementParsers.h
View File

@ -156,13 +156,28 @@ protected:
bool allow_function_parameters;
};
// Window definition (the thing that goes after OVER) for window function.
// Window reference (the thing that goes after OVER) for window function.
// Can be either window name or window definition.
class ParserWindowReference : public IParserBase
{
const char * getName() const override { return "window reference"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserWindowDefinition : public IParserBase
{
const char * getName() const override { return "window definition"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
// The WINDOW clause of a SELECT query that defines a list of named windows.
// Returns an ASTExpressionList of ASTWindowListElement's.
class ParserWindowList : public IParserBase
{
const char * getName() const override { return "WINDOW clause"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserCodecDeclarationList : public IParserBase
{
protected:

13
src/Parsers/ParserSelectQuery.cpp
View File

@ -39,6 +39,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
ParserKeyword s_with("WITH");
ParserKeyword s_totals("TOTALS");
ParserKeyword s_having("HAVING");
ParserKeyword s_window("WINDOW");
ParserKeyword s_order_by("ORDER BY");
ParserKeyword s_limit("LIMIT");
ParserKeyword s_settings("SETTINGS");
@ -71,6 +72,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
ASTPtr where_expression;
ASTPtr group_expression_list;
ASTPtr having_expression;
ASTPtr window_list;
ASTPtr order_expression_list;
ASTPtr limit_by_length;
ASTPtr limit_by_offset;
@ -203,6 +205,16 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
return false;
}
/// WINDOW clause
if (s_window.ignore(pos, expected))
{
ParserWindowList window_list_parser;
if (!window_list_parser.parse(pos, window_list, expected))
{
return false;
}
}
/// ORDER BY expr ASC|DESC COLLATE 'locale' list
if (s_order_by.ignore(pos, expected))
{
@ -374,6 +386,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
select_query->setExpression(ASTSelectQuery::Expression::WHERE, std::move(where_expression));
select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, std::move(group_expression_list));
select_query->setExpression(ASTSelectQuery::Expression::HAVING, std::move(having_expression));
select_query->setExpression(ASTSelectQuery::Expression::WINDOW, std::move(window_list));
select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_expression_list));
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, std::move(limit_by_offset));
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, std::move(limit_by_length));

5
src/Parsers/TokenIterator.cpp
View File

@ -4,12 +4,13 @@
namespace DB
{
UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token * last)
UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token last)
{
/// We have just two kind of parentheses: () and [].
UnmatchedParentheses stack;
for (TokenIterator it = begin; it.isValid() && &it.get() <= last; ++it)
for (TokenIterator it = begin;
it.isValid() && it->begin <= last.begin; ++it)
{
if (it->type == TokenType::OpeningRoundBracket || it->type == TokenType::OpeningSquareBracket)
{

2
src/Parsers/TokenIterator.h
View File

@ -80,6 +80,6 @@ public:
/// Returns positions of unmatched parentheses.
using UnmatchedParentheses = std::vector<Token>;
UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token * last);
UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token last);
}

97
src/Parsers/parseQuery.cpp
View File

@ -78,6 +78,10 @@ void writeQueryWithHighlightedErrorPositions(
for (size_t position_to_hilite_idx = 0; position_to_hilite_idx < num_positions_to_hilite; ++position_to_hilite_idx)
{
const char * current_position_to_hilite = positions_to_hilite[position_to_hilite_idx].begin;
assert(current_position_to_hilite < end);
assert(current_position_to_hilite >= begin);
out.write(pos, current_position_to_hilite - pos);
if (current_position_to_hilite == end)
@ -189,6 +193,10 @@ std::string getLexicalErrorMessage(
writeQueryAroundTheError(out, begin, end, hilite, &last_token, 1);
out << getErrorTokenDescription(last_token.type);
if (last_token.size())
{
out << ": '" << StringRef{last_token.begin, last_token.size()} << "'";
}
return out.str();
}
@ -217,8 +225,8 @@ std::string getUnmatchedParenthesesErrorMessage(
ASTPtr tryParseQuery(
IParser & parser,
const char * & pos,
const char * end,
const char * & _out_query_end, /* also query begin as input parameter */
const char * all_queries_end,
std::string & out_error_message,
bool hilite,
const std::string & query_description,
@ -226,7 +234,8 @@ ASTPtr tryParseQuery(
size_t max_query_size,
size_t max_parser_depth)
{
Tokens tokens(pos, end, max_query_size);
const char * query_begin = _out_query_end;
Tokens tokens(query_begin, all_queries_end, max_query_size);
IParser::Pos token_iterator(tokens, max_parser_depth);
if (token_iterator->isEnd()
@ -241,70 +250,90 @@ ASTPtr tryParseQuery(
//"
// Advance the position, so that we can use this parser for stream parsing
// even in presence of such queries.
pos = token_iterator->begin;
_out_query_end = token_iterator->begin;
return nullptr;
}
Expected expected;
ASTPtr res;
bool parse_res = parser.parse(token_iterator, res, expected);
Token last_token = token_iterator.max();
const bool parse_res = parser.parse(token_iterator, res, expected);
const auto last_token = token_iterator.max();
_out_query_end = last_token.end;
/// If parsed query ends at data for insertion. Data for insertion could be in any format and not necessary be lexical correct.
ASTInsertQuery * insert = nullptr;
if (parse_res)
insert = res->as<ASTInsertQuery>();
if (!(insert && insert->data))
// If parsed query ends at data for insertion. Data for insertion could be
// in any format and not necessary be lexical correct, so we can't perform
// most of the checks.
if (insert && insert->data)
{
/// Lexical error
if (last_token.isError())
if (!parse_res)
{
out_error_message = getLexicalErrorMessage(pos, end, last_token, hilite, query_description);
// Generic parse error.
out_error_message = getSyntaxErrorMessage(query_begin, all_queries_end,
last_token, expected, hilite, query_description);
return nullptr;
}
/// Unmatched parentheses
UnmatchedParentheses unmatched_parens = checkUnmatchedParentheses(TokenIterator(tokens), &last_token);
if (!unmatched_parens.empty())
{
out_error_message = getUnmatchedParenthesesErrorMessage(pos, end, unmatched_parens, hilite, query_description);
return nullptr;
}
return res;
}
// More granular checks for queries other than INSERT w/inline data.
/// Lexical error
if (last_token.isError())
{
out_error_message = getLexicalErrorMessage(query_begin, all_queries_end,
last_token, hilite, query_description);
return nullptr;
}
/// Unmatched parentheses
UnmatchedParentheses unmatched_parens = checkUnmatchedParentheses(TokenIterator(tokens), last_token);
if (!unmatched_parens.empty())
{
out_error_message = getUnmatchedParenthesesErrorMessage(query_begin,
all_queries_end, unmatched_parens, hilite, query_description);
return nullptr;
}
if (!parse_res)
{
/// Parse error.
out_error_message = getSyntaxErrorMessage(pos, end, last_token, expected, hilite, query_description);
/// Generic parse error.
out_error_message = getSyntaxErrorMessage(query_begin, all_queries_end,
last_token, expected, hilite, query_description);
return nullptr;
}
/// Excessive input after query. Parsed query must end with end of data or semicolon or data for INSERT.
if (!token_iterator->isEnd()
&& token_iterator->type != TokenType::Semicolon
&& !(insert && insert->data))
&& token_iterator->type != TokenType::Semicolon)
{
expected.add(pos, "end of query");
out_error_message = getSyntaxErrorMessage(pos, end, last_token, expected, hilite, query_description);
expected.add(last_token.begin, "end of query");
out_error_message = getSyntaxErrorMessage(query_begin, all_queries_end,
last_token, expected, hilite, query_description);
return nullptr;
}
// Skip the semicolon that might be left after parsing the VALUES format.
while (token_iterator->type == TokenType::Semicolon)
++token_iterator;
/// If multi-statements are not allowed, then after semicolon, there must be no non-space characters.
if (!allow_multi_statements
&& !token_iterator->isEnd()
&& !(insert && insert->data))
{
out_error_message = getSyntaxErrorMessage(pos, end, last_token, {}, hilite,
(query_description.empty() ? std::string() : std::string(". ")) + "Multi-statements are not allowed");
++token_iterator;
}
// If multi-statements are not allowed, then after semicolon, there must
// be no non-space characters.
if (!allow_multi_statements
&& !token_iterator->isEnd())
{
out_error_message = getSyntaxErrorMessage(query_begin, all_queries_end,
last_token, {}, hilite,
(query_description.empty() ? std::string() : std::string(". "))
+ "Multi-statements are not allowed");
return nullptr;
}
pos = token_iterator->begin;
return res;
}

2
src/Parsers/parseQuery.h
View File

@ -9,7 +9,7 @@ namespace DB
/// Parse query or set 'out_error_message'.
ASTPtr tryParseQuery(
IParser & parser,
const char * & pos, /// Moved to end of parsed fragment.
const char * & _out_query_end, // query start as input parameter, query end as output
const char * end,
std::string & out_error_message,
bool hilite,

10
src/Parsers/queryNormalization.h
View File

@ -126,9 +126,17 @@ inline void ALWAYS_INLINE normalizeQueryToPODArray(const char * begin, const cha
if (!prev_insignificant)
{
if (0 == num_literals_in_sequence)
res_data.push_back(' ');
{
// If it's leading whitespace, ignore it altogether.
if (token.begin != begin)
{
res_data.push_back(' ');
}
}
else
{
prev_whitespace = true;
}
}
prev_insignificant = true;
continue;

1
src/Parsers/ya.make
View File

@ -60,6 +60,7 @@ SRCS(
ASTTTLElement.cpp
ASTTablesInSelectQuery.cpp
ASTUserNameWithHost.cpp
ASTWindowDefinition.cpp
ASTWithAlias.cpp
ASTWithElement.cpp
CommonParsers.cpp

10
tests/clickhouse-test
View File

@ -570,7 +570,7 @@ def main(args):
if not check_server_started(args.client, args.server_check_retries):
raise Exception(
"Server is not responding. Cannot execute 'SELECT 1' query. \
Note: if you are using unbundled mode, you also have to specify -c option.")
Note: if you are using split build, you may have to specify -c option.")
build_flags = collect_build_flags(args.client)
if args.antlr:
@ -848,10 +848,10 @@ if __name__ == '__main__':
parser.add_argument('--tmp', help='Path to tmp dir')
parser.add_argument('-b', '--binary', default='clickhouse',
help='Path to clickhouse (if bundled, clickhouse-server otherwise) binary or name of binary in PATH')
help='Path to clickhouse (if monolithic build, clickhouse-server otherwise) binary or name of binary in PATH')
parser.add_argument('-c', '--client',
help='Path to clickhouse-client (if unbundled, useless otherwise) binary of name of binary in PATH')
help='Path to clickhouse-client (if split build, useless otherwise) binary of name of binary in PATH')
parser.add_argument('--extract_from_config', help='extract-from-config program')
parser.add_argument('--configclient', help='Client config (if you use not default ports)')
@ -932,11 +932,11 @@ if __name__ == '__main__':
if find_binary(args.binary + '-client'):
args.client = args.binary + '-client'
print("Using " + args.client + " as client program (expecting unbundled mode)")
print("Using " + args.client + " as client program (expecting split build)")
elif find_binary(args.binary):
args.client = args.binary + ' client'
print("Using " + args.client + " as client program (expecting bundled mode)")
print("Using " + args.client + " as client program (expecting monolithic build)")
else:
print("No 'clickhouse' or 'clickhouse-client' client binary found", file=sys.stderr)
parser.print_help()

2
tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh
View File

@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "DROP TABLE IF EXISTS tab_00651; CREATE TABLE tab_00651 (val UInt64) engine = Memory; SHOW CREATE TABLE tab_00651 format abcd; DESC tab_00651; DROP TABLE tab_00651;" ||: 2> /dev/null
${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "DROP TABLE IF EXISTS tab_00651; CREATE TABLE tab_00651 (val UInt64) engine = Memory; SHOW CREATE TABLE tab_00651 format abcd; DESC tab_00651; DROP TABLE tab_00651;" 2>/dev/null ||:

6
tests/queries/0_stateless/01091_num_threads.sql
View File

@ -8,7 +8,7 @@ WITH
(
SELECT query_id
FROM system.query_log
WHERE (query = 'WITH 01091 AS id SELECT 1;\n') AND (event_date >= (today() - 1))
WHERE (normalizeQuery(query) like normalizeQuery('WITH 01091 AS id SELECT 1;')) AND (event_date >= (today() - 1))
ORDER BY event_time DESC
LIMIT 1
) AS id
@ -23,7 +23,7 @@ WITH
(
SELECT query_id
FROM system.query_log
WHERE (query LIKE 'with 01091 as id select sum(number) from numbers(1000000);%') AND (event_date >= (today() - 1))
WHERE (normalizeQuery(query) = normalizeQuery('with 01091 as id select sum(number) from numbers(1000000);')) AND (event_date >= (today() - 1))
ORDER BY event_time DESC
LIMIT 1
) AS id
@ -38,7 +38,7 @@ WITH
(
SELECT query_id
FROM system.query_log
WHERE (query LIKE 'with 01091 as id select sum(number) from numbers_mt(1000000);%') AND (event_date >= (today() - 1))
WHERE (normalizeQuery(query) = normalizeQuery('with 01091 as id select sum(number) from numbers_mt(1000000);')) AND (event_date >= (today() - 1))
ORDER BY event_time DESC
LIMIT 1
) AS id

9
tests/queries/0_stateless/01475_read_subcolumns.reference
View File

@ -19,3 +19,12 @@ baz
0
1
2
====map====
['a','b']
['a','c']
['b','c']
[1,2]
[3,4]
[5,6]
4
4

19
tests/queries/0_stateless/01475_read_subcolumns.sql
View File

@ -42,3 +42,22 @@ SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'FileOpen')]
FROM system.query_log
WHERE (type = 'QueryFinish') AND (lower(query) LIKE lower('SELECT n.null FROM %t_nul%'))
AND event_time > now() - INTERVAL 10 SECOND AND current_database = currentDatabase();
SELECT '====map====';
SET allow_experimental_map_type = 1;
DROP TABLE IF EXISTS t_map;
CREATE TABLE t_map (m Map(String, UInt32)) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0;
INSERT INTO t_map VALUES (map('a', 1, 'b', 2)) (map('a', 3, 'c', 4)), (map('b', 5, 'c', 6));
--- will read 4 files: keys.bin, keys.mrk2, size0.bin, size0.mrk2
SYSTEM DROP MARK CACHE;
SELECT m.keys FROM t_map;
SYSTEM DROP MARK CACHE;
SELECT m.values FROM t_map;
SYSTEM FLUSH LOGS;
SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'FileOpen')]
FROM system.query_log
WHERE (type = 'QueryFinish') AND (lower(query) LIKE lower('SELECT m.% FROM %t_map%'))
AND event_time > now() - INTERVAL 10 SECOND AND current_database = currentDatabase();

24
tests/queries/0_stateless/01475_read_subcolumns_storages.reference
View File

@ -1,18 +1,18 @@
Log
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]
TinyLog
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]
Memory
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]
MergeTree ORDER BY tuple() SETTINGS min_bytes_for_compact_part='10M'
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]
MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part='10M'
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]
MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200)
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200
100 [1,2,3] [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] [1,NULL,2] ('foo',200) {'foo':1,'bar':42}
100 0 [1,2,3] 3 [[[1,2],[],[4]],[[5,6],[7,8]],[[]]] 3 [3,2,1] [[2,0,1],[2,2],[0]] [1,NULL,2] 3 [0,1,0] ('foo',200) foo 200 {'foo':1,'bar':42} ['foo','bar'] [1,42]

8
tests/queries/0_stateless/01475_read_subcolumns_storages.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
set -e
create_query="CREATE TABLE subcolumns(n Nullable(UInt32), a1 Array(UInt32),\
a2 Array(Array(Array(UInt32))), a3 Array(Nullable(UInt32)), t Tuple(s String, v UInt32))"
a2 Array(Array(Array(UInt32))), a3 Array(Nullable(UInt32)), t Tuple(s String, v UInt32), m Map(String, UInt32))"
# "StripeLog"
declare -a ENGINES=("Log" "TinyLog" "Memory" \
@ -18,8 +18,8 @@ declare -a ENGINES=("Log" "TinyLog" "Memory" \
for engine in "${ENGINES[@]}"; do
echo $engine
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS subcolumns"
$CLICKHOUSE_CLIENT --query "$create_query ENGINE = $engine"
$CLICKHOUSE_CLIENT --query "INSERT INTO subcolumns VALUES (100, [1, 2, 3], [[[1, 2], [], [4]], [[5, 6], [7, 8]], [[]]], [1, NULL, 2], ('foo', 200))"
$CLICKHOUSE_CLIENT --query "$create_query ENGINE = $engine" --allow_experimental_map_type 1
$CLICKHOUSE_CLIENT --query "INSERT INTO subcolumns VALUES (100, [1, 2, 3], [[[1, 2], [], [4]], [[5, 6], [7, 8]], [[]]], [1, NULL, 2], ('foo', 200), map('foo', 1, 'bar', 42))"
$CLICKHOUSE_CLIENT --query "SELECT * FROM subcolumns"
$CLICKHOUSE_CLIENT --query "SELECT n, n.null, a1, a1.size0, a2, a2.size0, a2.size1, a2.size2, a3, a3.size0, a3.null, t, t.s, t.v FROM subcolumns"
$CLICKHOUSE_CLIENT --query "SELECT n, n.null, a1, a1.size0, a2, a2.size0, a2.size1, a2.size2, a3, a3.size0, a3.null, t, t.s, t.v, m, m.keys, m.values FROM subcolumns"
done

12
tests/queries/0_stateless/01531_query_log_query_comment.sql
View File

@ -4,9 +4,17 @@ set log_queries_min_type='QUERY_FINISH';
set enable_global_with_statement=1;
select /* test=01531, enable_global_with_statement=0 */ 2;
system flush logs;
select count() from system.query_log where event_time >= now() - interval 5 minute and query = 'select /* test=01531, enable_global_with_statement=0 */ 2;\n';
select count() from system.query_log
where event_time >= now() - interval 5 minute
and query like '%select /* test=01531, enable_global_with_statement=0 */ 2%'
and current_database = currentDatabase()
;
set enable_global_with_statement=1;
select /* test=01531 enable_global_with_statement=1 */ 2;
system flush logs;
select count() from system.query_log where event_time >= now() - interval 5 minute and query = 'select /* test=01531 enable_global_with_statement=1 */ 2;\n';
select count() from system.query_log
where event_time >= now() - interval 5 minute
and query like '%select /* test=01531 enable_global_with_statement=1 */ 2%'
and current_database = currentDatabase()
;

27
tests/queries/0_stateless/01564_test_hint_woes.reference Normal file
View File

@ -0,0 +1,27 @@
-- { echo }
create table values_01564(
a int,
constraint c1 check a < 10) engine Memory;
-- client error hint after broken insert values
insert into values_01564 values ('f'); -- { clientError 6 }
insert into values_01564 values ('f'); -- { clientError 6 }
select 1;
1
insert into values_01564 values ('f'); -- { clientError 6 }
select nonexistent column; -- { serverError 47 }
select 1;
1
select nonexistent column; -- { serverError 47 }
-- server error hint after broken insert values (violated constraint)
insert into values_01564 values (11); -- { serverError 469 }
insert into values_01564 values (11); -- { serverError 469 }
select 1;
1
insert into values_01564 values (11); -- { serverError 469 }
select nonexistent column; -- { serverError 47 }
-- query after values on the same line
insert into values_01564 values (1); select 1;
select 1;
1
-- the return code must be zero after the final query has failed with expected error
insert into values_01564 values (11); -- { serverError 469 }

46
tests/queries/0_stateless/01564_test_hint_woes.sql Normal file
View File

@ -0,0 +1,46 @@
-- { echo }
create table values_01564(
a int,
constraint c1 check a < 10) engine Memory;
-- client error hint after broken insert values
insert into values_01564 values ('f'); -- { clientError 6 }
insert into values_01564 values ('f'); -- { clientError 6 }
select 1;
insert into values_01564 values ('f'); -- { clientError 6 }
select nonexistent column; -- { serverError 47 }
-- syntax error hint after broken insert values
insert into values_01564 this is bad syntax values ('f'); -- { clientError 62 }
insert into values_01564 this is bad syntax values ('f'); -- { clientError 62 }
select 1;
insert into values_01564 this is bad syntax values ('f'); -- { clientError 62 }
select nonexistent column; -- { serverError 47 }
-- server error hint after broken insert values (violated constraint)
insert into values_01564 values (11); -- { serverError 469 }
insert into values_01564 values (11); -- { serverError 469 }
select 1;
insert into values_01564 values (11); -- { serverError 469 }
select nonexistent column; -- { serverError 47 }
-- query after values on the same line
insert into values_01564 values (1); select 1;
-- even this works (not sure why we need it lol)
-- insert into values_01564 values (11) /*{ serverError 469 }*/; select 1;
-- syntax error, where the last token we can parse is long before the semicolon.
select this is too many words for an alias; -- { clientError 62 }
OPTIMIZE TABLE values_01564 DEDUPLICATE BY; -- { clientError 62 }
OPTIMIZE TABLE values_01564 DEDUPLICATE BY a EXCEPT a; -- { clientError 62 }
select 'a' || distinct one || 'c' from system.one; -- { clientError 62 }
-- the return code must be zero after the final query has failed with expected error
insert into values_01564 values (11); -- { serverError 469 }

149
tests/queries/0_stateless/01591_window_functions.reference
View File

@ -1,11 +1,8 @@
-- { echo }
set allow_experimental_window_functions = 1;
-- just something basic
select number, count() over (partition by intDiv(number, 3) order by number) from numbers(10);
-- proper calculation across blocks
0 1
1 2
2 3
@ -16,10 +13,8 @@ select number, count() over (partition by intDiv(number, 3) order by number) fro
7 2
8 3
9 1
-- proper calculation across blocks
select number, max(number) over (partition by intDiv(number, 3) order by number desc) from numbers(10) settings max_block_size = 2;
-- not a window function
2 2
1 2
0 2
@ -30,14 +25,10 @@ select number, max(number) over (partition by intDiv(number, 3) order by number
7 8
6 8
9 9
-- not a window function
select number, abs(number) over (partition by toString(intDiv(number, 3))) from numbers(10); -- { serverError 63 }
-- no partition by
select number, avg(number) over (order by number) from numbers(10);
-- no order by
0 0
1 0.5
2 1
@ -48,10 +39,8 @@ select number, avg(number) over (order by number) from numbers(10);
7 3.5
8 4
9 4.5
-- no order by
select number, quantileExact(number) over (partition by intDiv(number, 3)) from numbers(10);
-- can add an alias after window spec
0 0
1 1
2 1
@ -62,36 +51,28 @@ select number, quantileExact(number) over (partition by intDiv(number, 3)) from
7 7
8 7
9 9
-- can add an alias after window spec
select number, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10);
0 0
1 1
2 1
3 3
4 4
5 4
6 6
7 7
8 7
9 9
-- can't reference it yet -- the window functions are calculated at the
-- last stage of select, after all other functions.
0 0
1 1
2 1
3 3
4 4
5 4
6 6
7 7
8 7
9 9
select q * 10, quantileExact(number) over (partition by intDiv(number, 3)) q from numbers(10); -- { serverError 47 }
-- must work in WHERE if you wrap it in a subquery
select * from (select count(*) over () c from numbers(3)) where c > 0;
-- should work in ORDER BY
1
2
3
-- should work in ORDER BY
select number, max(number) over (partition by intDiv(number, 3) order by number desc) m from numbers(10) order by m desc, number;
-- also works in ORDER BY if you wrap it in a subquery
9 9
6 8
7 8
@ -102,43 +83,40 @@ select number, max(number) over (partition by intDiv(number, 3) order by number
0 2
1 2
2 2
-- also works in ORDER BY if you wrap it in a subquery
select * from (select count(*) over () c from numbers(3)) order by c;
1
2
3
-- Example with window function only in ORDER BY. Here we make a rank of all
-- numbers sorted descending, and then sort by this rank descending, and must get
-- the ascending order.
select * from (select * from numbers(5) order by rand()) order by count() over (order by number desc) desc;
0
1
2
3
select * from (select * from numbers(5) order by rand()) order by count() over (order by number desc) desc;
4
-- Aggregate functions as window function arguments. This query is semantically
-- the same as the above one, only we replace `number` with
-- `any(number) group by number` and so on.
select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc;
0
1
2
3
4
select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number) + 1) over (order by min(number) desc) desc;
-- some more simple cases w/aggregate functions
select sum(any(number)) over () from numbers(1);
0
select sum(any(number) + 1) over () from numbers(1);
1
select sum(any(number + 1)) over () from numbers(1);
1
-- different windows
-- an explain test would also be helpful, but it's too immature now and I don't
-- want to change reference all the time
0
1
2
3
4
select number, max(number) over (partition by intDiv(number, 3) order by number desc), count(number) over (partition by intDiv(number, 5) order by number) as m from numbers(31) order by number settings max_block_size = 2;
-- two functions over the same window
-- an explain test would also be helpful, but it's too immature now and I don't
-- want to change reference all the time
0 2 1
1 2 2
2 2 3
@ -170,10 +148,10 @@ select number, max(number) over (partition by intDiv(number, 3) order by number
28 29 4
29 29 5
30 30 1
-- two functions over the same window
-- an explain test would also be helpful, but it's too immature now and I don't
-- want to change reference all the time
select number, max(number) over (partition by intDiv(number, 3) order by number desc), count(number) over (partition by intDiv(number, 3) order by number desc) as m from numbers(7) order by number settings max_block_size = 2;
-- check that we can work with constant columns
0 2 3
1 2 2
2 2 1
@ -181,35 +159,26 @@ select number, max(number) over (partition by intDiv(number, 3) order by number
4 5 2
5 5 1
6 6 1
-- check that we can work with constant columns
select median(x) over (partition by x) from (select 1 x);
-- an empty window definition is valid as well
1
-- an empty window definition is valid as well
select groupArray(number) over () from numbers(3);
-- This one tests we properly process the window function arguments.
-- Seen errors like 'column `1` not found' from count(1).
[0]
[0,1]
[0,1,2]
-- This one tests we properly process the window function arguments.
-- Seen errors like 'column `1` not found' from count(1).
select count(1) over (), max(number + 1) over () from numbers(3);
-- Should work in DISTINCT
1 3
-- Should work in DISTINCT
select distinct sum(0) over () from numbers(2);
0
select distinct any(number) over () from numbers(2);
0
-- Various kinds of aliases are properly substituted into various parts of window
-- function definition.
0
with number + 1 as x select intDiv(number, 3) as y, sum(x + y) over (partition by y order by x) from numbers(7);
0 1
0 3
0 6
@ -217,3 +186,39 @@ with number + 1 as x select intDiv(number, 3) as y, sum(x + y) over (partition b
1 11
1 18
2 9
-- WINDOW clause
select 1 window w1 as ();
1
select sum(number) over w1, sum(number) over w2
from numbers(10)
window
w1 as (),
w2 as (partition by intDiv(number, 3))
;
0 0
1 1
3 3
6 3
10 7
15 12
21 6
28 13
36 21
45 9
select
sum(number) over w1,
sum(number) over (partition by intDiv(number, 3))
from numbers(10)
window
w1 as (partition by intDiv(number, 3))
;
0 0
1 1
3 3
3 3
7 7
12 12
6 6
13 13
21 21
9 9

24
tests/queries/0_stateless/01591_window_functions.sql
View File

@ -41,7 +41,11 @@ select * from (select * from numbers(5) order by rand()) order by count() over (
-- Aggregate functions as window function arguments. This query is semantically
-- the same as the above one, only we replace `number` with
-- `any(number) group by number` and so on.
select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number) + 1) over (order by min(number) desc) desc;
select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc;
-- some more simple cases w/aggregate functions
select sum(any(number)) over () from numbers(1);
select sum(any(number) + 1) over () from numbers(1);
select sum(any(number + 1)) over () from numbers(1);
-- different windows
-- an explain test would also be helpful, but it's too immature now and I don't
@ -70,3 +74,21 @@ select distinct any(number) over () from numbers(2);
-- Various kinds of aliases are properly substituted into various parts of window
-- function definition.
with number + 1 as x select intDiv(number, 3) as y, sum(x + y) over (partition by y order by x) from numbers(7);
-- WINDOW clause
select 1 window w1 as ();
select sum(number) over w1, sum(number) over w2
from numbers(10)
window
w1 as (),
w2 as (partition by intDiv(number, 3))
;
select
sum(number) over w1,
sum(number) over (partition by intDiv(number, 3))
from numbers(10)
window
w1 as (partition by intDiv(number, 3))
;

3
tests/tsan_suppressions.txt
View File

@ -1,2 +1 @@
# looks like a bug in clang-11 thread sanitizer, detects normal data race with random FD in this method
race:DB::LazyPipeFDs::close
# We have no suppressions!